diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..c22e36d8a3a87e1d530d174c7008ce63473d6535 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,95 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/run-u3lo36eq.wandb filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/0_eval_0_f2d24d1f27d7648055dc.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/3_eval_3_a45dc41fdb9f639ccef4.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/7_eval_7_d015da1851d414bb6628.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/episode_rosbags/episode_5_2025-01-18-19-13-04.bag filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/12_eval_12_9c363b6170975b374255.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/5_eval_5_c150bb925f103adcede9.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/17_eval_17_9f5ca32b8d4315bd9618.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/13_eval_13_c552251a3cda7642b754.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/4_eval_4_edd1b447fab3432974b4.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/16_eval_16_a29e67cd0c3e166bf43e.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/19_eval_19_98200c8b3f44031bae08.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/8_eval_8_9d16e439b84d1b213c76.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/10_eval_10_741429cbea5a900591e5.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/15_eval_15_3efd5b9c2b7508acddc5.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/2_eval_2_340d87e376b6dfa888fb.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/14_eval_14_58248ea2be38c14379cd.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/18_eval_18_fe8dc0d3e79f58b5302a.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/6_eval_6_438f8d27cd8ba8c28d6a.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/1_eval_1_f85e4baf2746d94aa0a7.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/11_eval_11_5d5f6eccd1ac8a9641c2.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/9_eval_9_d74f7b5c31b1fe3c2190.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/19_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/12_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/8_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/7_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/13_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/18_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/11_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/9_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/6_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/15_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/5_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/16_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/17_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/10_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/14_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/episode_rosbags/episode_12_2025-01-18-19-19-01.bag filter=lfs diff=lfs merge=lfs -text +u3lo36eq/episode_rosbags/episode_19_2025-01-18-19-26-02.bag filter=lfs diff=lfs merge=lfs -text +u3lo36eq/episode_rosbags/episode_8_2025-01-18-19-15-26.bag filter=lfs diff=lfs merge=lfs -text +u3lo36eq/episode_rosbags/episode_6_2025-01-18-19-13-48.bag filter=lfs diff=lfs merge=lfs -text +u3lo36eq/episode_rosbags/episode_16_2025-01-18-19-23-20.bag filter=lfs diff=lfs merge=lfs -text +u3lo36eq/episode_rosbags/episode_0_2025-01-18-19-09-22.bag filter=lfs diff=lfs merge=lfs -text +u3lo36eq/episode_rosbags/episode_17_2025-01-18-19-24-16.bag filter=lfs diff=lfs merge=lfs -text +u3lo36eq/episode_rosbags/episode_13_2025-01-18-19-19-43.bag filter=lfs diff=lfs merge=lfs -text +u3lo36eq/episode_rosbags/episode_14_2025-01-18-19-21-30.bag filter=lfs diff=lfs merge=lfs -text +u3lo36eq/episode_rosbags/episode_10_2025-01-18-19-17-15.bag filter=lfs diff=lfs merge=lfs -text +u3lo36eq/episode_rosbags/episode_3_2025-01-18-19-11-30.bag filter=lfs diff=lfs merge=lfs -text +u3lo36eq/episode_rosbags/episode_18_2025-01-18-19-25-14.bag filter=lfs diff=lfs merge=lfs -text +u3lo36eq/episode_rosbags/episode_1_2025-01-18-19-10-10.bag filter=lfs diff=lfs merge=lfs -text +u3lo36eq/episode_rosbags/episode_4_2025-01-18-19-12-21.bag filter=lfs diff=lfs merge=lfs -text +u3lo36eq/episode_rosbags/episode_11_2025-01-18-19-18-17.bag filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/episode_rosbags/episode_0_2025-01-18-20-46-53.bag filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/episode_rosbags/episode_18_2025-01-18-21-00-42.bag filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/episode_rosbags/episode_3_2025-01-18-20-49-03.bag filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/run-7sdjk1lj.wandb filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/15_eval_15_3c41565be23b9dc07e38.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/4_eval_4_c2149e115233e8c46adb.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/11_eval_11_c3bd4b9c68e23c764392.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/1_eval_1_f99205433ef4972a7f4b.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/episode_rosbags/episode_2_2025-01-18-20-48-24.bag filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/10_eval_10_e5b99072831fb49e316b.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/5_eval_5_b6ffac2c1558879bc8f9.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/16_eval_16_92f238d9751dde8bacfd.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/6_eval_6_b37677ea4de36477338d.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/3_eval_3_b0e12a1c923453fa2db6.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/18_eval_18_831456511316b3120131.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/0_eval_0_8f82b238c966cdc59610.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/17_eval_17_cff7eb6bb7d10755b32e.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/8_eval_8_9db24d2dbee878841c0a.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/19_eval_19_cc714c15f81700a6ca67.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/13_eval_13_cf7080346708b3da4f52.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/12_eval_12_941e17d058fd7b746278.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/9_eval_9_bdaa1599f7ed02f8d4c8.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/2_eval_2_f1446b1ad582539ee197.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/14_eval_14_38839f2b6468bf8a5f05.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/7_eval_7_09f63e9126c49c6ebd71.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/eval_video/19_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/eval_video/12_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u3lo36eq/episode_rosbags/episode_2_2025-01-18-19-10-48.bag filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/eval_video/8_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/eval_video/7_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/eval_video/18_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/eval_video/9_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7sdjk1lj/episode_rosbags/episode_5_2025-01-18-20-50-25.bag filter=lfs diff=lfs merge=lfs -text diff --git a/7sdjk1lj/episode_rosbags/aligned_depth_to_color_K.npy b/7sdjk1lj/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/7sdjk1lj/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/7sdjk1lj/episode_rosbags/cam_tf_world.npy b/7sdjk1lj/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/7sdjk1lj/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/7sdjk1lj/episode_rosbags/color_K.npy b/7sdjk1lj/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/7sdjk1lj/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/7sdjk1lj/episode_rosbags/depth_K.npy b/7sdjk1lj/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/7sdjk1lj/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/7sdjk1lj/episode_rosbags/episode_0_2025-01-18-20-46-53.bag b/7sdjk1lj/episode_rosbags/episode_0_2025-01-18-20-46-53.bag new file mode 100644 index 0000000000000000000000000000000000000000..edd951c01d2d300b288c63148ea33665997b3bf4 --- /dev/null +++ b/7sdjk1lj/episode_rosbags/episode_0_2025-01-18-20-46-53.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:371599e8eec9dfe2064499b171745dbe1dced2f564074255589fb21ec2647d4c +size 1411347596 diff --git a/7sdjk1lj/episode_rosbags/episode_18_2025-01-18-21-00-42.bag b/7sdjk1lj/episode_rosbags/episode_18_2025-01-18-21-00-42.bag new file mode 100644 index 0000000000000000000000000000000000000000..4fdc23b1914baaf6579f1e4c036aafdab85876e6 --- /dev/null +++ b/7sdjk1lj/episode_rosbags/episode_18_2025-01-18-21-00-42.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ff1a410b7dbcbc8c91386814315cf4760457f6cac3f765f732a6f5687b44ef8 +size 1413620003 diff --git a/7sdjk1lj/episode_rosbags/episode_2_2025-01-18-20-48-24.bag b/7sdjk1lj/episode_rosbags/episode_2_2025-01-18-20-48-24.bag new file mode 100644 index 0000000000000000000000000000000000000000..dd761ae90177fe37e07405bef69536b943e8e68d --- /dev/null +++ b/7sdjk1lj/episode_rosbags/episode_2_2025-01-18-20-48-24.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a0ae81495b79a788b82a51ca919a3a97df7e5c45a99a410049ede019e290526 +size 1409359857 diff --git a/7sdjk1lj/episode_rosbags/episode_3_2025-01-18-20-49-03.bag b/7sdjk1lj/episode_rosbags/episode_3_2025-01-18-20-49-03.bag new file mode 100644 index 0000000000000000000000000000000000000000..2e6c3f9a3ff3f4e20e4a2c81c9d8af9849988fa8 --- /dev/null +++ b/7sdjk1lj/episode_rosbags/episode_3_2025-01-18-20-49-03.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c32e1126b3c7e58e8e6bae5e5b03913c6a2fb95ff10c0a7e23ca2eeb04bfac50 +size 1410834478 diff --git a/7sdjk1lj/episode_rosbags/episode_5_2025-01-18-20-50-25.bag b/7sdjk1lj/episode_rosbags/episode_5_2025-01-18-20-50-25.bag new file mode 100644 index 0000000000000000000000000000000000000000..835d265267f299dff7fc228a3494a89f6ccb9260 --- /dev/null +++ b/7sdjk1lj/episode_rosbags/episode_5_2025-01-18-20-50-25.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cf1b75bbf85d44b3b8d8ae01c398a3274822ff4072e1ca1d78f3a53530b047e +size 1409835419 diff --git a/7sdjk1lj/eval_robot.log b/7sdjk1lj/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..65582eada1ca6f6f90f41dbdd64dba516705cc90 --- /dev/null +++ b/7sdjk1lj/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-18 20:46:06,180][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-18 20:46:06,183][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/7sdjk1lj/eval_video/12_eval.mp4 b/7sdjk1lj/eval_video/12_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..77f9752e08f1c663e469f0bded510e24b3443e84 --- /dev/null +++ b/7sdjk1lj/eval_video/12_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:941e17d058fd7b7462780a3767dd17bcf47dbde226e47dc9793ba48f5b7c3175 +size 1149306 diff --git a/7sdjk1lj/eval_video/18_eval.mp4 b/7sdjk1lj/eval_video/18_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6ef0b60a5d21f2a0fd323c87cb2f7252612bf4c7 --- /dev/null +++ b/7sdjk1lj/eval_video/18_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:831456511316b3120131a6f259078ab1d77677226790bfa29dc32863b69e1a2f +size 1503368 diff --git a/7sdjk1lj/eval_video/19_eval.mp4 b/7sdjk1lj/eval_video/19_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..38f2275245b26b9ea2266fa089d27b217cbaddf2 --- /dev/null +++ b/7sdjk1lj/eval_video/19_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc714c15f81700a6ca671303f08983a4f63b247b5594bbeb385a946ba8ce35f4 +size 1573555 diff --git a/7sdjk1lj/eval_video/3_eval.mp4 b/7sdjk1lj/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e37db5ea0d9d38ce53ddb5a60cb109004f927428 --- /dev/null +++ b/7sdjk1lj/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0e12a1c923453fa2db69a1f45ec1a121ab7b26d16dcb037170480cfcfeacbaf +size 1146532 diff --git a/7sdjk1lj/eval_video/4_eval.mp4 b/7sdjk1lj/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e14956217b05c2947c0a82970aa8b1877ceb5f30 --- /dev/null +++ b/7sdjk1lj/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2149e115233e8c46adb595ad1e67e57c08904c4eb9560b94d48b032a3e82cd0 +size 1200313 diff --git a/7sdjk1lj/eval_video/7_eval.mp4 b/7sdjk1lj/eval_video/7_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..41d12668d5d2b6f403459711751997dd9de5a16d --- /dev/null +++ b/7sdjk1lj/eval_video/7_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09f63e9126c49c6ebd71be8e8336e3403d6ed934dc7cb8186565e6aa14214662 +size 1238849 diff --git a/7sdjk1lj/eval_video/8_eval.mp4 b/7sdjk1lj/eval_video/8_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f29b3e32f3e4346972896809b15529485c45311d --- /dev/null +++ b/7sdjk1lj/eval_video/8_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9db24d2dbee878841c0adadd92b979ed83f33bd62249f56b1630a875df739652 +size 1273617 diff --git a/7sdjk1lj/eval_video/9_eval.mp4 b/7sdjk1lj/eval_video/9_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68fa2a91fa1eb882dee86e6b9bbe0ce7f550e055 --- /dev/null +++ b/7sdjk1lj/eval_video/9_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdaa1599f7ed02f8d4c847b92de72ce6ff375db54a425253f139d1cc8fb440a3 +size 1275582 diff --git a/7sdjk1lj/tb/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 b/7sdjk1lj/tb/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 new file mode 100644 index 0000000000000000000000000000000000000000..34cc539619bd58397f2d6765ddca986b5ee49787 --- /dev/null +++ b/7sdjk1lj/tb/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a52c9728aa2667499b72e684d6f053f7015c86f675bd22508d0ea5ff81c76fb0 +size 22784 diff --git a/7sdjk1lj/wandb/debug-internal.log b/7sdjk1lj/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b1d7680372f012b87ede6700e8ce52ac9315cb77 --- /dev/null +++ b/7sdjk1lj/wandb/debug-internal.log @@ -0,0 +1,2179 @@ +2025-01-18 20:46:12,103 INFO StreamThr :2573666 [internal.py:wandb_internal():85] W&B internal server running at pid: 2573666, started at: 2025-01-18 20:46:12.102899 +2025-01-18 20:46:12,104 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status +2025-01-18 20:46:12,115 INFO WriterThread:2573666 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/run-7sdjk1lj.wandb +2025-01-18 20:46:12,115 DEBUG SenderThread:2573666 [sender.py:send():379] send: header +2025-01-18 20:46:12,123 DEBUG SenderThread:2573666 [sender.py:send():379] send: run +2025-01-18 20:46:12,392 INFO SenderThread:2573666 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files +2025-01-18 20:46:12,392 INFO SenderThread:2573666 [sender.py:_start_run_threads():1188] run started: 7sdjk1lj with start time 1737251172.104356 +2025-01-18 20:46:12,412 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: check_version +2025-01-18 20:46:12,413 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: check_version +2025-01-18 20:46:12,457 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: run_start +2025-01-18 20:46:12,462 DEBUG HandlerThread:2573666 [system_info.py:__init__():26] System info init +2025-01-18 20:46:12,462 DEBUG HandlerThread:2573666 [system_info.py:__init__():41] System info init done +2025-01-18 20:46:12,462 INFO HandlerThread:2573666 [system_monitor.py:start():194] Starting system monitor +2025-01-18 20:46:12,462 INFO SystemMonitor:2573666 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-18 20:46:12,462 INFO HandlerThread:2573666 [system_monitor.py:probe():214] Collecting system info +2025-01-18 20:46:12,462 INFO SystemMonitor:2573666 [interfaces.py:start():188] Started cpu monitoring +2025-01-18 20:46:12,463 INFO SystemMonitor:2573666 [interfaces.py:start():188] Started disk monitoring +2025-01-18 20:46:12,463 INFO SystemMonitor:2573666 [interfaces.py:start():188] Started gpu monitoring +2025-01-18 20:46:12,464 INFO SystemMonitor:2573666 [interfaces.py:start():188] Started memory monitoring +2025-01-18 20:46:12,465 INFO SystemMonitor:2573666 [interfaces.py:start():188] Started network monitoring +2025-01-18 20:46:12,475 DEBUG HandlerThread:2573666 [system_info.py:probe():152] Probing system +2025-01-18 20:46:12,477 DEBUG HandlerThread:2573666 [system_info.py:_probe_git():137] Probing git +2025-01-18 20:46:12,481 DEBUG HandlerThread:2573666 [system_info.py:_probe_git():145] Probing git done +2025-01-18 20:46:12,481 DEBUG HandlerThread:2573666 [system_info.py:probe():200] Probing system done +2025-01-18 20:46:12,481 DEBUG HandlerThread:2573666 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-19T01:46:12.475557', 'startedAt': '2025-01-19T01:46:12.087278', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3380.134625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4563.213, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 631.7242088317871}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-18 20:46:12,481 INFO HandlerThread:2573666 [system_monitor.py:probe():224] Finished collecting system info +2025-01-18 20:46:12,481 INFO HandlerThread:2573666 [system_monitor.py:probe():227] Publishing system info +2025-01-18 20:46:12,481 DEBUG HandlerThread:2573666 [system_info.py:_save_code():44] Saving code +2025-01-18 20:46:12,487 DEBUG HandlerThread:2573666 [system_info.py:_save_code():67] Saving code done +2025-01-18 20:46:12,487 DEBUG HandlerThread:2573666 [system_info.py:_save_patches():84] Saving git patches +2025-01-18 20:46:12,586 DEBUG HandlerThread:2573666 [system_info.py:_save_patches():126] Saving git patches done +2025-01-18 20:46:12,587 INFO HandlerThread:2573666 [system_monitor.py:probe():229] Finished publishing system info +2025-01-18 20:46:12,590 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:46:12,590 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-18 20:46:12,590 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-18 20:46:12,591 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-18 20:46:12,591 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-18 20:46:12,853 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: python_packages +2025-01-18 20:46:12,853 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: python_packages +2025-01-18 20:46:12,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:46:12,854 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:46:12,867 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/8at10yt1-wandb-metadata.json +2025-01-18 20:46:12,870 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/wafsayyp-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 20:46:12,872 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/7dvjyqlt-code/FISH/eval_robot.py +2025-01-18 20:46:12,889 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/tm9i5og1-diff.patch +2025-01-18 20:46:12,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:12,897 INFO HandlerThread:2573666 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/tb" + save: true +} +control { + relay_id: "ffe869506cfe4ddda48be41fbc39527d" +} +_info { + stream_id: "7sdjk1lj" +} + +2025-01-18 20:46:12,897 DEBUG HandlerThread:2573666 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-18 20:46:12,956 DEBUG SenderThread:2573666 [sender.py:send():379] send: telemetry +2025-01-18 20:46:12,956 DEBUG SenderThread:2573666 [sender.py:send():379] send: tbrecord +2025-01-18 20:46:12,958 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:46:12,958 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 with policy live +2025-01-18 20:46:13,151 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/67lsmr8t-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:46:13,394 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/requirements.txt +2025-01-18 20:46:13,394 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/code/FISH/eval_robot.py +2025-01-18 20:46:13,394 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-metadata.json +2025-01-18 20:46:13,394 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 20:46:13,394 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:46:13,394 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:13,395 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/diff.patch +2025-01-18 20:46:13,395 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/code +2025-01-18 20:46:13,395 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/code/FISH +2025-01-18 20:46:13,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:14,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:15,393 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:15,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:16,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:17,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:46:17,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:18,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:19,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:20,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:21,856 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:22,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:46:22,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:23,856 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:24,856 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:25,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:26,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:27,853 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:46:27,854 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:46:27,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:27,942 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:46:28,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:29,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:30,269 DEBUG SenderThread:2573666 [sender.py:send():379] send: config +2025-01-18 20:46:30,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:31,397 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:31,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:32,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:33,398 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:33,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:46:33,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:34,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:35,399 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:35,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:36,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:37,400 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:37,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:38,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:39,400 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:39,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:46:39,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:40,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:41,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:42,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:46:42,854 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:46:42,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:43,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:44,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:44,978 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:46:45,402 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/config.yaml +2025-01-18 20:46:45,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:46,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:47,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:48,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:49,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:50,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:51,019 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:46:51,404 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:51,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:52,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:53,404 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:53,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:54,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:55,405 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:55,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:56,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:57,007 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:46:57,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:46:57,854 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:46:57,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:58,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:59,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:00,862 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:01,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:02,011 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:02,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:03,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:04,862 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:05,862 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:06,862 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:07,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:07,862 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:08,863 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:09,863 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:10,863 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:11,863 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:12,465 DEBUG SystemMonitor:2573666 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-18 20:47:12,465 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:47:12,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:47:12,854 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:47:12,901 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:12,954 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:13,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:14,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:15,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:16,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:17,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:18,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:18,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:19,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:20,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:21,661 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:47:21,661 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:47:21,661 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_8f82b238c966cdc59610.mp4 with policy now +2025-01-18 20:47:21,662 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:47:21,663 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:47:21,663 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:47:21,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:21,912 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/z47dc6lj-media/videos/eval/0_eval_0_8f82b238c966cdc59610.mp4 +2025-01-18 20:47:22,413 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:47:22,414 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/0_eval_0_8f82b238c966cdc59610.mp4 +2025-01-18 20:47:22,414 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos +2025-01-18 20:47:22,414 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:47:22,414 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media +2025-01-18 20:47:22,715 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/el3cbdz8-media/videos/eval/0_eval_0_8f82b238c966cdc59610.mp4 +2025-01-18 20:47:22,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:23,414 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:47:23,865 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:24,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:24,865 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:25,865 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:26,865 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:27,416 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:47:27,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:47:27,854 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:47:27,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:28,865 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:29,416 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:47:29,865 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:29,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:30,865 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:31,866 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:32,866 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:33,866 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:34,866 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:35,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:35,866 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:36,885 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:37,419 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:47:37,670 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/f8f0zrm2-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:47:37,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:38,867 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:39,420 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:47:39,867 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:40,867 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:41,789 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:41,867 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:42,468 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:47:42,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:47:42,854 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:47:42,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:43,422 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:47:43,867 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:44,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:45,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:46,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:46,929 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:47,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:48,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:49,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:50,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:51,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:52,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:52,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:53,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:54,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:55,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:56,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:57,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:47:57,854 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:47:57,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:58,010 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:58,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:59,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:00,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:01,870 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:02,870 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:03,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:03,870 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:04,870 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:05,870 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:06,871 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:07,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:08,648 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:48:08,648 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f99205433ef4972a7f4b.mp4 with policy now +2025-01-18 20:48:08,649 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:48:08,651 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:48:08,651 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:48:08,652 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:48:08,872 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:08,917 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/6bg37292-media/videos/eval/1_eval_1_f99205433ef4972a7f4b.mp4 +2025-01-18 20:48:09,430 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:09,430 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:48:09,430 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/1_eval_1_f99205433ef4972a7f4b.mp4 +2025-01-18 20:48:09,430 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:48:09,661 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:09,725 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/pnystu9r-media/videos/eval/1_eval_1_f99205433ef4972a7f4b.mp4 +2025-01-18 20:48:09,873 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:10,872 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:11,430 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:11,873 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:12,466 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:48:12,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:48:12,855 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:48:12,873 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:13,431 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:13,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:14,875 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:14,884 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:15,873 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:16,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:17,432 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:17,880 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:18,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:19,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:20,380 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:20,433 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:48:20,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:21,322 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:48:21,322 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:48:21,322 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:48:21,434 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:21,434 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:48:21,875 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:22,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:23,435 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:23,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:24,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:25,435 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:25,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:25,937 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:26,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:27,436 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:27,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:48:27,855 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:48:27,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:28,875 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:29,875 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:30,875 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:31,875 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:31,914 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:32,875 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:33,876 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:34,876 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:35,876 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:36,876 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:37,149 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:37,439 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:37,876 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:38,876 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:39,876 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:40,876 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:41,877 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:42,467 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:48:42,467 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:42,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:48:42,855 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:48:42,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:43,877 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:44,877 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:45,877 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:46,877 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:47,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:47,878 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:48,878 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:49,878 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:50,878 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:51,885 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:52,233 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:48:52,234 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:48:52,234 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_f1446b1ad582539ee197.mp4 with policy now +2025-01-18 20:48:52,234 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:48:52,234 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:48:52,235 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:48:52,444 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:48:52,444 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/2_eval_2_f1446b1ad582539ee197.mp4 +2025-01-18 20:48:52,444 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:48:52,505 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/vizqbk2o-media/videos/eval/2_eval_2_f1446b1ad582539ee197.mp4 +2025-01-18 20:48:52,770 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/d31v9196-media/videos/eval/2_eval_2_f1446b1ad582539ee197.mp4 +2025-01-18 20:48:52,865 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:52,883 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:53,444 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:53,889 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:54,445 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:54,883 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:55,905 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:56,887 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:57,446 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:57,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:48:57,860 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:48:57,901 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:57,957 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:58,885 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:59,446 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:59,889 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:00,150 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:49:00,150 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:49:00,151 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:49:00,447 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:49:00,447 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:49:00,698 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/55qdj1rm-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:49:00,890 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:01,447 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:01,895 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:02,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:03,448 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:03,843 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:03,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:04,893 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:05,449 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:05,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:06,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:07,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:08,862 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:08,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:09,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:10,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:11,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:12,467 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:49:12,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:49:12,856 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:49:12,901 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:13,895 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:14,203 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:14,895 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:15,452 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:15,895 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:16,895 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:17,895 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:18,895 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:19,862 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:19,896 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:20,896 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:21,896 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:22,896 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:23,896 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:24,896 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:25,445 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:25,456 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:25,896 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:26,896 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:27,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:49:27,856 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:49:27,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:28,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:29,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:30,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:30,986 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:31,910 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:32,400 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:49:32,400 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_b0e12a1c923453fa2db6.mp4 with policy now +2025-01-18 20:49:32,404 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:49:32,405 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:49:32,405 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:49:32,405 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:49:32,458 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:49:32,458 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/3_eval_3_b0e12a1c923453fa2db6.mp4 +2025-01-18 20:49:32,459 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:49:32,661 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/d49sz6f5-media/videos/eval/3_eval_3_b0e12a1c923453fa2db6.mp4 +2025-01-18 20:49:32,901 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:32,914 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/vrmwoqgq-media/videos/eval/3_eval_3_b0e12a1c923453fa2db6.mp4 +2025-01-18 20:49:33,459 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:33,904 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:34,459 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:34,910 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:35,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:36,923 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:36,985 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:37,460 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:37,932 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:38,909 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:39,916 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:40,206 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:49:40,206 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:49:40,206 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:49:40,461 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:49:40,461 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:49:40,920 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:41,461 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:41,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:42,468 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:49:42,468 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:42,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:49:42,859 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:49:42,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:43,915 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:44,914 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:45,463 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:45,914 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:46,915 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:47,463 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:47,745 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:47,915 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:48,915 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:49,915 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:50,915 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:51,916 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:52,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:52,916 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:53,916 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:54,916 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:55,916 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:56,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:57,856 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:49:57,856 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:49:57,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:57,961 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:58,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:59,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:00,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:01,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:02,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:03,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:03,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:04,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:05,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:06,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:07,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:08,871 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:08,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:09,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:10,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:11,472 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:11,924 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:12,468 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:50:12,516 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:50:12,516 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_c2149e115233e8c46adb.mp4 with policy now +2025-01-18 20:50:12,520 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:50:12,520 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:50:12,520 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:50:12,520 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:50:12,799 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/gwq92w9i-media/videos/eval/4_eval_4_c2149e115233e8c46adb.mp4 +2025-01-18 20:50:12,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:50:12,860 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:50:12,924 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:13,472 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:13,473 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:50:13,473 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/4_eval_4_c2149e115233e8c46adb.mp4 +2025-01-18 20:50:13,473 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:50:13,770 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/0eeyohxv-media/videos/eval/4_eval_4_c2149e115233e8c46adb.mp4 +2025-01-18 20:50:13,930 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:13,981 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:14,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:15,938 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:16,935 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:17,474 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:17,932 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:18,930 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:19,005 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:19,944 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:20,933 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:21,475 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:21,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:22,258 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:50:22,258 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:50:22,258 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:50:22,476 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:50:22,476 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:50:22,724 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/u65nxnty-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:50:22,933 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:23,476 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:23,934 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:24,137 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:24,931 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:25,477 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:25,937 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:26,937 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:27,478 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:27,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:50:27,858 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:50:27,934 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:28,934 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:29,479 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:29,934 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:29,969 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:30,935 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:31,935 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:32,935 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:33,935 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:34,935 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:35,873 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:35,935 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:36,935 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:37,935 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:38,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:39,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:40,873 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:40,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:41,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:42,469 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:50:42,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:50:42,858 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:50:42,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:43,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:44,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:45,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:45,965 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:46,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:47,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:48,937 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:49,937 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:50,938 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:51,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:51,938 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:52,939 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:53,939 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:54,946 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:55,488 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:55,812 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:50:55,812 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_b6ffac2c1558879bc8f9.mp4 with policy now +2025-01-18 20:50:55,814 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:50:55,815 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:50:55,815 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:50:55,815 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:50:55,942 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:56,076 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/vg3pqp14-media/videos/eval/5_eval_5_b6ffac2c1558879bc8f9.mp4 +2025-01-18 20:50:56,488 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:56,488 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:50:56,488 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/5_eval_5_b6ffac2c1558879bc8f9.mp4 +2025-01-18 20:50:56,489 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:50:56,823 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/ar0jnjz5-media/videos/eval/5_eval_5_b6ffac2c1558879bc8f9.mp4 +2025-01-18 20:50:56,953 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:57,489 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:57,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:57,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:50:57,864 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:50:57,954 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:58,945 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:59,955 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:00,948 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:01,490 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:01,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:02,881 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:02,961 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:03,491 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:03,956 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:04,952 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:05,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:06,308 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:51:06,309 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:51:06,309 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:51:06,492 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:51:06,492 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:51:06,956 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:07,492 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:07,889 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:07,962 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:08,964 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:09,957 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:10,957 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:11,494 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:11,957 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:12,469 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:51:12,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:51:12,859 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:51:12,957 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:12,958 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:13,957 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:14,958 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:15,958 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:16,958 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:17,958 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:18,881 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:18,958 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:19,958 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:20,958 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:21,958 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:22,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:23,881 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:23,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:24,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:25,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:26,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:27,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:51:27,859 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:51:27,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:28,957 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:28,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:29,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:30,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:31,960 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:32,960 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:33,960 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:34,881 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:34,960 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:35,960 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:36,965 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:37,503 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:37,690 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:51:37,690 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_b37677ea4de36477338d.mp4 with policy now +2025-01-18 20:51:37,693 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:51:37,694 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:51:37,694 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:51:37,694 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:51:37,946 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/o099li45-media/videos/eval/6_eval_6_b37677ea4de36477338d.mp4 +2025-01-18 20:51:37,969 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:38,504 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:51:38,504 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/6_eval_6_b37677ea4de36477338d.mp4 +2025-01-18 20:51:38,504 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:51:38,811 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/v1rd2hyy-media/videos/eval/6_eval_6_b37677ea4de36477338d.mp4 +2025-01-18 20:51:38,971 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:39,504 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:39,890 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:39,973 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:40,980 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:41,981 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:42,470 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:51:42,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:51:42,869 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:51:42,974 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:43,506 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:43,983 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:44,979 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:45,257 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:45,507 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:45,993 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:46,983 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:47,508 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:51:47,746 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/q49voqrf-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:51:48,007 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:48,361 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:51:48,362 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:51:48,362 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:51:48,508 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:51:49,002 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:49,509 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:50,002 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:50,533 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:50,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:51,996 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:52,995 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:53,510 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:53,990 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:54,990 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:55,511 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:55,923 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:55,990 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:56,990 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:57,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:51:57,860 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:51:57,990 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:58,990 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:59,990 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:00,990 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:01,011 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:01,990 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:02,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:03,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:04,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:05,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:06,883 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:06,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:07,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:08,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:09,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:10,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:11,517 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:11,901 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:11,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:12,470 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:52:12,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:52:12,859 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:52:12,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:13,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:14,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:15,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:16,933 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:16,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:17,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:18,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:19,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:20,996 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:21,520 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:21,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:22,018 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:22,097 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:52:22,098 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:52:22,098 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_09f63e9126c49c6ebd71.mp4 with policy now +2025-01-18 20:52:22,098 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:52:22,098 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:52:22,099 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:52:22,361 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/hywi46bh-media/videos/eval/7_eval_7_09f63e9126c49c6ebd71.mp4 +2025-01-18 20:52:22,521 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:52:22,521 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/7_eval_7_09f63e9126c49c6ebd71.mp4 +2025-01-18 20:52:22,521 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:52:22,799 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/wik1nr23-media/videos/eval/7_eval_7_09f63e9126c49c6ebd71.mp4 +2025-01-18 20:52:23,012 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:23,521 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:24,003 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:24,522 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:25,007 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:26,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:27,019 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:27,523 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:27,689 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:27,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:52:27,868 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:52:28,027 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:29,016 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:29,524 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:30,025 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:30,413 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:52:30,414 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:52:30,461 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:52:30,525 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:52:30,525 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:52:31,025 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:31,525 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:32,026 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:32,892 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:33,034 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:33,526 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:34,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:35,021 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:35,527 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:36,029 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:37,035 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:38,022 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:38,887 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:39,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:39,528 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:40,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:41,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:42,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:42,470 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:52:42,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:52:42,861 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:52:43,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:43,953 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:44,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:45,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:46,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:47,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:48,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:49,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:49,887 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:50,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:51,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:52,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:53,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:54,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:54,887 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:55,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:56,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:57,025 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:57,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:52:57,861 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:52:58,025 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:59,025 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:59,932 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:00,025 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:01,025 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:02,025 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:03,025 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:04,026 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:05,020 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:05,049 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:05,468 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:53:05,468 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_9db24d2dbee878841c0a.mp4 with policy now +2025-01-18 20:53:05,469 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:53:05,469 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:53:05,469 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:53:05,471 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:53:05,538 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:05,538 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:53:05,538 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/8_eval_8_9db24d2dbee878841c0a.mp4 +2025-01-18 20:53:05,538 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:53:05,750 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/3np193h9-media/videos/eval/8_eval_8_9db24d2dbee878841c0a.mp4 +2025-01-18 20:53:06,028 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/pq0cnz3m-media/videos/eval/8_eval_8_9db24d2dbee878841c0a.mp4 +2025-01-18 20:53:06,031 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:07,039 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:07,539 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:08,049 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:09,031 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:10,043 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:10,173 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:11,042 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:11,540 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:12,047 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:12,471 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:53:12,867 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:53:12,867 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:53:13,055 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:13,541 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:14,059 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:15,061 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:15,973 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:16,076 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:16,469 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:53:16,470 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:53:16,470 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:53:16,542 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:53:16,542 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:53:16,787 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/gzopmazz-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:53:17,060 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:17,542 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:18,069 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:19,059 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:19,543 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:20,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:21,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:21,133 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:21,544 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:22,079 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:23,075 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:24,075 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:25,076 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:25,545 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:26,076 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:26,302 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:27,076 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:27,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:53:27,861 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:53:28,076 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:29,076 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:30,076 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:31,077 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:31,971 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:32,077 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:33,077 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:34,077 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:35,077 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:36,077 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:37,077 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:37,906 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:38,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:39,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:40,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:41,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:42,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:42,471 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:53:42,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:53:42,862 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:53:43,004 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:43,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:44,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:45,079 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:46,079 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:47,079 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:48,079 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:48,906 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:49,079 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:50,079 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:51,083 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:51,439 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:53:51,439 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_bdaa1599f7ed02f8d4c8.mp4 with policy now +2025-01-18 20:53:51,443 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:53:51,444 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:53:51,444 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:53:51,444 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:53:51,555 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:51,556 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:53:51,556 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/9_eval_9_bdaa1599f7ed02f8d4c8.mp4 +2025-01-18 20:53:51,556 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:53:51,706 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/452df9xa-media/videos/eval/9_eval_9_bdaa1599f7ed02f8d4c8.mp4 +2025-01-18 20:53:51,997 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/ce6xzab0-media/videos/eval/9_eval_9_bdaa1599f7ed02f8d4c8.mp4 +2025-01-18 20:53:52,097 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:52,556 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:53,095 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:53,556 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:53,932 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:54,102 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:55,092 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:56,099 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:57,119 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:57,558 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:57,884 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:53:57,884 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:53:58,100 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:59,094 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:59,113 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:59,558 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:00,120 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:01,116 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:02,111 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:03,112 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:04,118 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:04,527 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:54:04,527 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:54:04,527 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:04,527 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:54:04,560 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:54:04,560 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:54:05,117 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:05,561 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:06,121 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:07,153 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:08,122 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:09,123 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:09,562 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:10,123 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:10,286 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:11,123 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:11,563 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:12,123 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:12,472 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:54:12,862 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:54:12,863 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:54:13,123 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:14,123 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:15,123 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:15,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:16,123 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:17,123 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:17,565 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:18,124 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:19,124 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:20,124 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:20,920 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:21,124 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:22,124 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:23,124 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:24,124 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:25,124 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:25,921 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:26,124 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:27,125 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:27,862 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:54:27,863 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:54:28,125 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:29,126 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:30,126 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:31,002 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:31,126 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:32,126 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:33,126 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:34,126 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:35,127 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:35,572 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:36,156 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:36,190 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:36,501 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:54:36,501 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_e5b99072831fb49e316b.mp4 with policy now +2025-01-18 20:54:36,522 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:54:36,523 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:54:36,523 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:54:36,523 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:54:36,573 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:54:36,573 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/10_eval_10_e5b99072831fb49e316b.mp4 +2025-01-18 20:54:36,573 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:54:36,765 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/8hpk3irw-media/videos/eval/10_eval_10_e5b99072831fb49e316b.mp4 +2025-01-18 20:54:37,023 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/oaullsv2-media/videos/eval/10_eval_10_e5b99072831fb49e316b.mp4 +2025-01-18 20:54:37,142 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:37,573 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:38,150 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:38,573 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:39,145 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:40,168 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:41,154 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:41,225 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:41,575 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:42,159 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:42,472 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:54:42,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:54:42,897 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:54:43,172 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:44,159 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:45,159 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:45,576 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:46,184 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:46,254 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:47,179 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:48,190 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:49,195 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:50,207 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:50,579 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:54:50,585 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:54:50,585 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:54:50,585 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:54:50,830 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/nmt2nr1s-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:54:51,194 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:51,285 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:51,579 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:51,580 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:54:52,185 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:53,191 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:53,580 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:54,209 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:55,183 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:55,581 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:56,183 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:56,666 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:57,183 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:57,863 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:54:57,863 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:54:58,183 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:59,183 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:00,184 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:01,184 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:01,961 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:02,184 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:03,184 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:04,184 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:05,184 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:05,585 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:06,184 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:07,184 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:07,198 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:08,185 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:09,185 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:09,587 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:10,185 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:11,185 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:11,588 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:12,185 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:12,473 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:55:12,474 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:12,863 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:55:12,863 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:55:13,185 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:13,589 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:14,186 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:15,186 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:15,590 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:16,186 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:17,186 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:17,841 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:18,186 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:19,186 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:19,592 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:20,187 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:21,187 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:22,217 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:22,649 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:55:22,649 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_c3bd4b9c68e23c764392.mp4 with policy now +2025-01-18 20:55:22,657 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:55:22,657 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:55:22,658 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:55:22,658 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:55:22,901 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/ljnzpbu7-media/videos/eval/11_eval_11_c3bd4b9c68e23c764392.mp4 +2025-01-18 20:55:22,941 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:23,189 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:23,594 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:23,594 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:55:23,594 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/11_eval_11_c3bd4b9c68e23c764392.mp4 +2025-01-18 20:55:23,594 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:55:23,897 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/6a0fcxbo-media/videos/eval/11_eval_11_c3bd4b9c68e23c764392.mp4 +2025-01-18 20:55:24,222 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:25,212 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:26,228 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:27,217 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:27,595 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:27,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:55:27,894 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:55:27,974 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:28,224 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:29,234 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:30,244 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:31,248 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:31,597 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:31,597 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:55:31,635 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:55:31,635 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:55:31,641 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:55:32,240 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:32,597 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:55:33,244 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:33,597 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:33,953 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:34,233 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:35,241 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:35,598 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:36,241 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:37,241 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:37,599 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:38,241 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:39,242 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:39,941 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:40,241 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:41,241 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:42,242 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:42,473 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:55:42,867 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:55:42,868 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:55:43,242 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:44,242 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:44,995 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:45,242 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:46,242 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:47,243 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:48,243 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:49,243 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:50,243 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:50,941 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:51,243 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:52,243 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:53,243 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:54,243 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:55,244 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:55,942 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:56,244 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:57,244 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:57,867 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:55:57,868 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:55:58,244 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:59,244 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:00,244 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:01,191 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:01,244 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:01,609 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:02,285 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:03,261 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:56:03,261 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_941e17d058fd7b746278.mp4 with policy now +2025-01-18 20:56:03,276 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:03,284 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:56:03,285 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:56:03,285 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:56:03,285 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:56:03,523 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/ltonb2ce-media/videos/eval/12_eval_12_941e17d058fd7b746278.mp4 +2025-01-18 20:56:03,610 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:03,610 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:56:03,610 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/12_eval_12_941e17d058fd7b746278.mp4 +2025-01-18 20:56:03,611 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:56:03,873 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/wstfjh2r-media/videos/eval/12_eval_12_941e17d058fd7b746278.mp4 +2025-01-18 20:56:04,264 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:05,281 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:05,611 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:06,286 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:06,372 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:07,294 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:07,612 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:08,312 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:09,318 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:10,303 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:11,316 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:11,613 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:12,109 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:12,317 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:12,474 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:56:12,882 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:56:12,883 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:56:13,316 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:14,330 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:14,615 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:56:14,687 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:56:14,687 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:56:14,688 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:56:15,349 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:15,615 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:15,615 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:56:16,341 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:17,343 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:17,349 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:17,616 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:18,321 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:19,352 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:19,617 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:20,332 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:21,357 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:22,349 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:22,445 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:23,351 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:24,339 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:25,333 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:25,619 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:26,333 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:27,333 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:27,620 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:27,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:56:27,869 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:56:28,028 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:28,333 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:29,333 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:30,333 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:31,333 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:32,334 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:33,334 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:33,622 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:33,964 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:34,334 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:35,334 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:36,334 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:37,337 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:38,334 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:38,964 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:39,335 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:39,625 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:40,335 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:41,335 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:42,335 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:42,474 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:56:42,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:56:42,869 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:56:43,335 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:43,626 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:44,137 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:44,335 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:45,335 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:46,336 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:47,335 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:48,336 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:49,336 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:49,629 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:49,965 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:50,336 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:51,336 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:52,357 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:53,346 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:53,549 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:56:53,549 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_cf7080346708b3da4f52.mp4 with policy now +2025-01-18 20:56:53,565 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:56:53,566 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:56:53,566 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:56:53,566 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:56:53,631 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:53,631 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:56:53,631 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/13_eval_13_cf7080346708b3da4f52.mp4 +2025-01-18 20:56:53,631 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:56:53,813 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/6dfva5eu-media/videos/eval/13_eval_13_cf7080346708b3da4f52.mp4 +2025-01-18 20:56:54,080 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/aisplhnz-media/videos/eval/13_eval_13_cf7080346708b3da4f52.mp4 +2025-01-18 20:56:54,353 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:55,351 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:55,604 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:55,632 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:56,348 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:57,356 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:57,632 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:57,886 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:56:57,887 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:56:58,387 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:59,373 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:00,350 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:01,205 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:01,353 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:01,634 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:02,355 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:03,383 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:04,379 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:05,371 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:06,262 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:06,367 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:06,636 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:57:06,752 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:57:06,752 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:57:06,753 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:57:06,897 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/xnwnqnjg-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:57:07,374 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:07,636 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:07,637 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:57:08,376 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:09,388 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:09,637 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:10,385 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:11,392 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:11,597 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:11,638 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:12,391 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:12,475 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:57:12,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:57:12,894 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:57:13,397 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:14,390 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:15,401 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:16,405 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:17,388 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:17,515 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:17,640 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:18,388 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:19,388 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:20,388 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:21,389 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:22,389 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:22,984 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:23,389 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:24,390 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:25,390 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:26,390 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:27,390 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:27,877 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:57:27,877 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:57:27,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:28,390 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:29,391 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:30,391 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:31,391 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:32,391 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:33,391 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:33,984 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:34,391 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:35,392 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:36,392 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:37,392 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:38,392 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:38,984 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:39,392 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:40,392 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:41,392 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:42,393 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:42,478 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:57:42,877 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:57:42,878 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:57:43,393 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:43,652 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:44,035 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:44,418 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:44,577 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:57:44,577 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_38839f2b6468bf8a5f05.mp4 with policy now +2025-01-18 20:57:44,588 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:57:44,588 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:57:44,588 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:57:44,588 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:57:44,653 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:57:44,653 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/14_eval_14_38839f2b6468bf8a5f05.mp4 +2025-01-18 20:57:44,653 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:57:44,837 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/8niptex4-media/videos/eval/14_eval_14_38839f2b6468bf8a5f05.mp4 +2025-01-18 20:57:45,098 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/o2zm5s4w-media/videos/eval/14_eval_14_38839f2b6468bf8a5f05.mp4 +2025-01-18 20:57:45,422 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:45,653 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:46,423 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:47,441 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:47,654 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:48,462 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:49,385 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:49,441 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:49,655 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:50,451 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:51,481 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:52,465 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:53,449 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:53,656 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:54,470 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:55,038 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:55,463 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:56,470 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:57,474 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:57,904 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:57:57,904 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:57:58,477 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:59,506 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:00,391 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:00,485 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:00,659 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:58:00,812 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:58:00,812 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:58:00,813 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:58:01,497 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:01,660 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:58:01,660 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:58:02,495 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:03,505 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:03,661 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:58:04,514 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:05,514 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:05,661 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:58:05,777 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:06,514 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:07,530 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:08,514 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:09,522 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:10,520 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:11,033 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:11,524 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:12,475 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:58:12,516 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:12,881 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:58:12,881 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:58:13,516 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:13,665 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:58:14,516 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:15,516 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:16,516 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:17,008 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:17,516 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:18,516 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:19,517 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:20,517 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:21,517 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:22,010 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:22,517 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:23,517 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:24,517 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:25,517 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:26,517 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:27,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:27,881 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:58:27,881 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:58:27,950 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:28,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:29,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:30,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:31,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:32,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:33,009 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:33,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:34,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:35,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:36,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:37,519 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:38,519 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:39,009 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:39,519 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:40,535 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:41,018 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:58:41,018 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_3c41565be23b9dc07e38.mp4 with policy now +2025-01-18 20:58:41,028 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:58:41,029 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:58:41,029 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:58:41,030 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:58:41,526 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:41,677 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:58:41,677 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/15_eval_15_3c41565be23b9dc07e38.mp4 +2025-01-18 20:58:41,677 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:58:41,759 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/jbxcrffu-media/videos/eval/15_eval_15_3c41565be23b9dc07e38.mp4 +2025-01-18 20:58:42,310 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/fv1ikth9-media/videos/eval/15_eval_15_3c41565be23b9dc07e38.mp4 +2025-01-18 20:58:42,476 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:58:42,538 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:42,677 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:58:42,911 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:58:42,911 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:58:43,558 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:43,678 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:58:44,085 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:44,532 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:45,558 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:46,576 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:46,679 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:58:47,557 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:48,575 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:49,555 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:50,013 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:50,566 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:50,680 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:58:51,586 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:52,579 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:53,589 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:54,594 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:55,016 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:55,608 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:56,594 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:57,609 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:57,912 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:58:57,912 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:58:58,584 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:59,588 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:00,059 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:00,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:00,883 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:59:00,883 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:59:00,884 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:59:01,627 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:01,685 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:59:01,686 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:59:02,647 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:02,686 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:03,653 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:04,648 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:04,686 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:05,247 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:05,616 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:06,617 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:06,687 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:07,617 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:08,617 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:08,688 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:09,617 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:10,617 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:11,022 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:11,617 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:12,476 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:59:12,617 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:12,891 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:59:12,891 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:59:13,617 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:14,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:15,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:16,022 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:16,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:17,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:18,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:19,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:20,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:20,694 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:21,338 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:21,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:22,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:23,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:24,619 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:25,619 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:26,619 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:27,022 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:27,619 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:27,891 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:59:27,892 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:59:28,620 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:29,620 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:30,620 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:31,620 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:32,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:32,627 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:33,660 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:33,968 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:59:33,968 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_92f238d9751dde8bacfd.mp4 with policy now +2025-01-18 20:59:34,001 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:59:34,002 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:59:34,002 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:59:34,002 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:59:34,350 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/9jbv8ta0-media/videos/eval/16_eval_16_92f238d9751dde8bacfd.mp4 +2025-01-18 20:59:34,669 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:34,700 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:34,700 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:59:34,700 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/16_eval_16_92f238d9751dde8bacfd.mp4 +2025-01-18 20:59:34,700 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:59:35,130 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/77836vkw-media/videos/eval/16_eval_16_92f238d9751dde8bacfd.mp4 +2025-01-18 20:59:35,661 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:35,700 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:36,662 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:36,700 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:37,120 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:37,684 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:38,679 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:39,676 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:40,681 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:40,702 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:41,686 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:42,477 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:59:42,477 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:42,693 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:42,703 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:42,910 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:59:42,910 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:59:43,719 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:44,703 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:45,725 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:46,705 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:59:46,739 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:46,939 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:59:46,939 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:59:46,940 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:59:46,965 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/qjx6oikp-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:59:47,706 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:59:47,706 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:59:47,708 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:47,753 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:48,706 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:48,714 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:49,714 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:50,707 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:50,725 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:51,742 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:52,708 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:52,729 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:52,809 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:53,736 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:54,748 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:55,730 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:56,722 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:57,761 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:57,846 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:57,895 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:59:57,895 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:59:58,711 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:58,715 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:59,715 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:00,712 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:00,715 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:01,715 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:02,715 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:03,071 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:03,715 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:04,715 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:05,715 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:06,714 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:06,716 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:07,716 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:08,071 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:08,716 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:09,716 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:10,716 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:11,716 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:12,477 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 21:00:12,716 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:12,895 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:00:12,895 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:00:13,632 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:13,716 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:14,717 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:15,716 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:16,717 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:16,719 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:17,717 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:18,717 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:19,071 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:19,717 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:20,717 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:21,717 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:22,717 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:23,718 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:24,073 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:24,720 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:25,730 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:25,903 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 21:00:25,903 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_cff7eb6bb7d10755b32e.mp4 with policy now +2025-01-18 21:00:25,913 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:00:25,914 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 21:00:25,914 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:00:25,914 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:00:26,279 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/rax0mwa4-media/videos/eval/17_eval_17_cff7eb6bb7d10755b32e.mp4 +2025-01-18 21:00:26,723 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:26,723 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:00:26,723 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/17_eval_17_cff7eb6bb7d10755b32e.mp4 +2025-01-18 21:00:26,723 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 21:00:26,734 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:27,156 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/lwbly7sr-media/videos/eval/17_eval_17_cff7eb6bb7d10755b32e.mp4 +2025-01-18 21:00:27,787 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:27,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:00:27,936 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:00:28,724 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:28,738 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:29,108 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:29,757 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:30,761 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:31,762 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:32,725 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:32,748 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:33,793 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:34,726 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:34,756 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:35,072 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:35,745 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:36,753 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:37,768 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:38,006 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 21:00:38,006 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:00:38,007 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:00:38,728 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:00:38,729 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 21:00:38,761 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:39,759 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:40,729 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:40,772 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:40,993 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:41,766 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:42,478 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 21:00:42,794 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:42,912 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:00:42,912 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:00:43,805 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:44,731 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:44,778 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:45,800 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:46,075 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:46,794 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:47,784 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:48,783 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:49,790 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:50,803 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:51,101 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:51,789 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:52,784 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:53,784 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:54,735 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:54,784 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:55,784 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:56,784 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:57,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:57,784 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:57,902 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:00:57,902 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:00:58,784 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:59,785 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:00,785 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:01,785 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:02,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:02,785 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:03,785 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:04,785 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:05,785 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:06,785 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:07,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:07,786 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:08,786 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:09,786 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:10,786 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:11,786 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:12,478 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 21:01:12,478 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:12,786 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:12,902 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:01:12,902 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:01:13,786 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:14,787 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:15,787 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:16,787 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:17,787 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:18,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:18,787 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:19,788 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:20,390 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 21:01:20,390 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_831456511316b3120131.mp4 with policy now +2025-01-18 21:01:20,401 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:01:20,401 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 21:01:20,401 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:01:20,401 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:01:20,747 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:01:20,747 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:01:20,747 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/18_eval_18_831456511316b3120131.mp4 +2025-01-18 21:01:20,747 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 21:01:20,765 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/1622zx39-media/videos/eval/18_eval_18_831456511316b3120131.mp4 +2025-01-18 21:01:20,808 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:21,177 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/jdd06vsr-media/videos/eval/18_eval_18_831456511316b3120131.mp4 +2025-01-18 21:01:21,747 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:01:21,808 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:22,748 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:01:22,856 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:23,096 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:23,748 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:01:23,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:24,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:25,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:26,750 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:01:26,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:27,841 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:27,902 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:01:27,902 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:01:28,144 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:28,751 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:01:28,887 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:29,751 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 21:01:29,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:30,071 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 21:01:30,071 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:01:30,072 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:01:30,752 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:01:30,752 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 21:01:30,910 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:31,915 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:32,753 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:01:32,922 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:33,886 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:34,115 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:34,885 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:35,889 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:36,755 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:01:36,915 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:37,913 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:38,940 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:39,133 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:39,941 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:40,913 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:41,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:42,479 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 21:01:42,927 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:42,946 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:01:42,946 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:01:43,924 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:44,922 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:44,979 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:45,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:46,760 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:01:46,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:47,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:48,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:49,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:50,114 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:50,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:51,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:52,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:53,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:54,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:55,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:56,114 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:56,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:57,902 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:01:57,903 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:01:57,945 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:58,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:59,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:00,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:01,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:02,114 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:02,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:03,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:04,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:05,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:06,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:07,114 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:07,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:08,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:09,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:10,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:11,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:12,152 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:12,479 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 21:02:12,948 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:12,949 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:02:12,949 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:02:13,935 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 21:02:13,935 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_cc714c15f81700a6ca67.mp4 with policy now +2025-01-18 21:02:13,939 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:13,963 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:02:13,963 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 21:02:13,963 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:13,964 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:14,334 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/pq6hgnnl-media/videos/eval/19_eval_19_cc714c15f81700a6ca67.mp4 +2025-01-18 21:02:14,773 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:02:14,773 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:02:14,773 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/19_eval_19_cc714c15f81700a6ca67.mp4 +2025-01-18 21:02:14,773 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 21:02:14,951 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:15,188 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/iwmah4tp-media/videos/eval/19_eval_19_cc714c15f81700a6ca67.mp4 +2025-01-18 21:02:15,773 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:02:15,942 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:16,774 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:02:16,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:17,675 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:17,993 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:18,983 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:19,997 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:20,776 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:02:20,988 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:21,986 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:22,777 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:02:23,019 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:23,146 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:24,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:25,069 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:26,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:26,778 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 21:02:27,022 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:27,140 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 21:02:27,141 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:27,141 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:27,779 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:02:27,949 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:02:27,949 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:02:28,040 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:28,121 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 21:02:28,121 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 21:02:28,351 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:02:28,626 INFO wandb-upload_1:2573666 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4oyofpyi +2025-01-18 21:02:28,713 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 21:02:28,722 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:02:28,779 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:02:28,780 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 21:02:28,780 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/table/eval/total_success_rate_ci_21_7c95301d486f1fc093c2.table.json +2025-01-18 21:02:28,780 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/table/eval/success_rate_ci_20_626687eda62aee1b0486.table.json +2025-01-18 21:02:28,780 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media +2025-01-18 21:02:28,780 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/table/eval +2025-01-18 21:02:28,780 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/table +2025-01-18 21:02:29,013 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:29,028 INFO SenderThread:2573666 [sender.py:send_request_log_artifact():1525] logged artifact run-7sdjk1lj-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MjAyMDY3MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzEwMDk1Mg==', 'latestArtifact': None}} +2025-01-18 21:02:29,028 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 21:02:29,029 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_626687eda62aee1b0486.table.json with policy now +2025-01-18 21:02:29,029 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:29,029 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 21:02:29,029 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,030 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,031 DEBUG SenderThread:2573666 [sender.py:send():379] send: telemetry +2025-01-18 21:02:29,032 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,034 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,034 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,035 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,035 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,035 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,036 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,036 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,036 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,037 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,037 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,037 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,037 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,038 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,038 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,039 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,039 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,039 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,040 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,040 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,040 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,041 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,041 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 21:02:29,277 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/7n3d5rvk-media/table/eval/success_rate_ci_20_626687eda62aee1b0486.table.json +2025-01-18 21:02:29,279 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/o4blrnq1-media/table/eval/total_success_rate_ci_21_7c95301d486f1fc093c2.table.json +2025-01-18 21:02:29,301 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/84uo7ur2-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 21:02:29,419 INFO wandb-upload_1:2573666 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpu2xsesb8 +2025-01-18 21:02:29,572 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/zewd7psa-media/table/eval/success_rate_ci_20_626687eda62aee1b0486.table.json +2025-01-18 21:02:29,780 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:02:29,838 INFO SenderThread:2573666 [sender.py:send_request_log_artifact():1525] logged artifact run-7sdjk1lj-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MjAyMDY3Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzEwMDk1Mw==', 'latestArtifact': None}} +2025-01-18 21:02:29,838 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 21:02:29,838 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_7c95301d486f1fc093c2.table.json with policy now +2025-01-18 21:02:29,838 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 21:02:29,838 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,839 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:30,013 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:30,096 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/mfwizwuz-media/table/eval/total_success_rate_ci_21_7c95301d486f1fc093c2.table.json +2025-01-18 21:02:30,781 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:02:30,781 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:02:31,013 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:32,013 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:33,013 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:34,013 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:34,135 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:35,014 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:36,014 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:37,014 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:38,014 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:39,014 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:39,135 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:40,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:41,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:42,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:42,479 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 21:02:42,915 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:02:42,916 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:02:43,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:44,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:44,135 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:45,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:46,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:47,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:48,016 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:49,016 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:50,016 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:50,135 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:51,016 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:52,016 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:53,016 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:54,017 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:55,017 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:55,138 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:55,793 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/config.yaml +2025-01-18 21:02:56,017 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:57,017 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:57,916 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:02:57,916 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:02:58,017 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:59,017 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:00,017 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:01,018 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:01,136 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:03:02,018 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:03,018 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:03,582 DEBUG SenderThread:2573666 [sender.py:send():379] send: exit +2025-01-18 21:03:03,582 INFO SenderThread:2573666 [sender.py:send_exit():586] handling exit code: 0 +2025-01-18 21:03:03,582 INFO SenderThread:2573666 [sender.py:send_exit():588] handling runtime: 1011 +2025-01-18 21:03:03,582 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:03:03,582 INFO SenderThread:2573666 [sender.py:send_exit():594] send defer +2025-01-18 21:03:03,582 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:03,582 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-18 21:03:03,583 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:03,583 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-18 21:03:03,583 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 1 +2025-01-18 21:03:03,583 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:03,583 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-18 21:03:03,583 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:03,583 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-18 21:03:03,583 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 2 +2025-01-18 21:03:03,583 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:03,583 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-18 21:03:03,583 INFO HandlerThread:2573666 [system_monitor.py:finish():203] Stopping system monitor +2025-01-18 21:03:03,584 DEBUG SystemMonitor:2573666 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-18 21:03:03,584 DEBUG SystemMonitor:2573666 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-18 21:03:03,584 INFO HandlerThread:2573666 [interfaces.py:finish():200] Joined cpu monitor +2025-01-18 21:03:03,585 INFO HandlerThread:2573666 [interfaces.py:finish():200] Joined disk monitor +2025-01-18 21:03:03,637 INFO HandlerThread:2573666 [interfaces.py:finish():200] Joined gpu monitor +2025-01-18 21:03:03,637 INFO HandlerThread:2573666 [interfaces.py:finish():200] Joined memory monitor +2025-01-18 21:03:03,637 INFO HandlerThread:2573666 [interfaces.py:finish():200] Joined network monitor +2025-01-18 21:03:03,638 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:03,638 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-18 21:03:03,638 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 3 +2025-01-18 21:03:03,638 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:03,638 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-18 21:03:03,643 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 21:03:03,643 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:03,643 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-18 21:03:03,643 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 4 +2025-01-18 21:03:03,643 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:03,643 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-18 21:03:03,797 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:03:04,798 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:03:10,151 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:03:10,152 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:03:10,152 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: keepalive +2025-01-18 21:03:10,153 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:10,153 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-18 21:03:10,153 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 5 +2025-01-18 21:03:10,153 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:03:10,153 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 21:03:10,153 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:03:10,153 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:10,153 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-18 21:03:10,154 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:03:10,154 DEBUG SenderThread:2573666 [sender.py:send():379] send: summary +2025-01-18 21:03:10,158 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:03:10,158 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:10,158 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-18 21:03:10,158 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 6 +2025-01-18 21:03:10,159 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:10,159 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-18 21:03:10,159 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:10,159 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-18 21:03:10,159 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 7 +2025-01-18 21:03:10,159 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:03:10,159 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:10,159 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-18 21:03:10,159 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:10,159 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-18 21:03:10,583 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:03:10,801 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:03:13,018 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 8 +2025-01-18 21:03:13,018 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:03:13,018 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:13,018 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-18 21:03:13,019 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:13,019 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-18 21:03:13,019 INFO SenderThread:2573666 [job_builder.py:build():440] Attempting to build job artifact +2025-01-18 21:03:13,022 INFO SenderThread:2573666 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-18 21:03:13,024 INFO SenderThread:2573666 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-18 21:03:13,039 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 9 +2025-01-18 21:03:13,039 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:13,039 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-18 21:03:13,039 DEBUG SenderThread:2573666 [sender.py:send():379] send: artifact +2025-01-18 21:03:13,404 INFO wandb-upload_2:2573666 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpnfp61zup +2025-01-18 21:03:13,406 INFO wandb-upload_3:2573666 [upload_job.py:push():85] Skipped uploading /tmp/tmp3mer6rfj/wandb-job.json +2025-01-18 21:03:13,519 INFO wandb-upload_1:2573666 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprfodh2rt +2025-01-18 21:03:13,584 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:03:13,803 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:03:14,226 INFO SenderThread:2573666 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1MjAyMTE2MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1MTk1NDMyOQ==', 'versionIndex': 449}}} +2025-01-18 21:03:14,226 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:14,227 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-18 21:03:14,227 INFO SenderThread:2573666 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-18 21:03:14,804 INFO SenderThread:2573666 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files +2025-01-18 21:03:14,804 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/config.yaml config.yaml +2025-01-18 21:03:14,804 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 21:03:14,804 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json wandb-summary.json +2025-01-18 21:03:14,804 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/requirements.txt requirements.txt +2025-01-18 21:03:14,804 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-metadata.json wandb-metadata.json +2025-01-18 21:03:14,804 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 21:03:14,804 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log output.log +2025-01-18 21:03:14,806 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/diff.patch diff.patch +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/2_eval_2_f1446b1ad582539ee197.mp4 media/videos/eval/2_eval_2_f1446b1ad582539ee197.mp4 +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/14_eval_14_38839f2b6468bf8a5f05.mp4 media/videos/eval/14_eval_14_38839f2b6468bf8a5f05.mp4 +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/7_eval_7_09f63e9126c49c6ebd71.mp4 media/videos/eval/7_eval_7_09f63e9126c49c6ebd71.mp4 +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/10_eval_10_e5b99072831fb49e316b.mp4 media/videos/eval/10_eval_10_e5b99072831fb49e316b.mp4 +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/4_eval_4_c2149e115233e8c46adb.mp4 media/videos/eval/4_eval_4_c2149e115233e8c46adb.mp4 +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/5_eval_5_b6ffac2c1558879bc8f9.mp4 media/videos/eval/5_eval_5_b6ffac2c1558879bc8f9.mp4 +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/3_eval_3_b0e12a1c923453fa2db6.mp4 media/videos/eval/3_eval_3_b0e12a1c923453fa2db6.mp4 +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/17_eval_17_cff7eb6bb7d10755b32e.mp4 media/videos/eval/17_eval_17_cff7eb6bb7d10755b32e.mp4 +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/19_eval_19_cc714c15f81700a6ca67.mp4 media/videos/eval/19_eval_19_cc714c15f81700a6ca67.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/9_eval_9_bdaa1599f7ed02f8d4c8.mp4 media/videos/eval/9_eval_9_bdaa1599f7ed02f8d4c8.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/8_eval_8_9db24d2dbee878841c0a.mp4 media/videos/eval/8_eval_8_9db24d2dbee878841c0a.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/16_eval_16_92f238d9751dde8bacfd.mp4 media/videos/eval/16_eval_16_92f238d9751dde8bacfd.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/12_eval_12_941e17d058fd7b746278.mp4 media/videos/eval/12_eval_12_941e17d058fd7b746278.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/1_eval_1_f99205433ef4972a7f4b.mp4 media/videos/eval/1_eval_1_f99205433ef4972a7f4b.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/11_eval_11_c3bd4b9c68e23c764392.mp4 media/videos/eval/11_eval_11_c3bd4b9c68e23c764392.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/15_eval_15_3c41565be23b9dc07e38.mp4 media/videos/eval/15_eval_15_3c41565be23b9dc07e38.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/13_eval_13_cf7080346708b3da4f52.mp4 media/videos/eval/13_eval_13_cf7080346708b3da4f52.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/0_eval_0_8f82b238c966cdc59610.mp4 media/videos/eval/0_eval_0_8f82b238c966cdc59610.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/18_eval_18_831456511316b3120131.mp4 media/videos/eval/18_eval_18_831456511316b3120131.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/6_eval_6_b37677ea4de36477338d.mp4 media/videos/eval/6_eval_6_b37677ea4de36477338d.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/table/eval/success_rate_ci_20_626687eda62aee1b0486.table.json media/table/eval/success_rate_ci_20_626687eda62aee1b0486.table.json +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/table/eval/total_success_rate_ci_21_7c95301d486f1fc093c2.table.json media/table/eval/total_success_rate_ci_21_7c95301d486f1fc093c2.table.json +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 10 +2025-01-18 21:03:14,813 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:03:14,813 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:14,813 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-18 21:03:14,813 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:14,813 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-18 21:03:14,813 INFO SenderThread:2573666 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 21:03:14,986 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/requirements.txt +2025-01-18 21:03:14,993 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:03:15,002 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/config.yaml +2025-01-18 21:03:15,021 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:03:15,221 INFO Thread-11 (_thread_body):2573666 [sender.py:transition_state():614] send defer: 11 +2025-01-18 21:03:15,222 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:15,222 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-18 21:03:15,222 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:15,222 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-18 21:03:15,222 INFO SenderThread:2573666 [file_pusher.py:join():175] waiting for file pusher +2025-01-18 21:03:15,225 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 12 +2025-01-18 21:03:15,225 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:15,225 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-18 21:03:15,225 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:15,225 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-18 21:03:15,225 INFO SenderThread:2573666 [file_stream.py:finish():601] file stream finish called +2025-01-18 21:03:15,297 INFO SenderThread:2573666 [file_stream.py:finish():605] file stream finish is done +2025-01-18 21:03:15,297 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 13 +2025-01-18 21:03:15,297 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:15,297 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-18 21:03:15,298 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:15,298 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-18 21:03:15,298 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 14 +2025-01-18 21:03:15,298 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:15,298 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-18 21:03:15,298 DEBUG SenderThread:2573666 [sender.py:send():379] send: final +2025-01-18 21:03:15,298 DEBUG SenderThread:2573666 [sender.py:send():379] send: footer +2025-01-18 21:03:15,298 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:15,298 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-18 21:03:15,299 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:03:15,299 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: server_info +2025-01-18 21:03:15,299 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:03:15,299 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:03:15,299 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: server_info +2025-01-18 21:03:15,300 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: get_summary +2025-01-18 21:03:15,301 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-18 21:03:15,302 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:15,367 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:03:15,368 INFO MainThread:2573666 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-18 21:03:15,368 INFO MainThread:2573666 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-18 21:03:15,369 INFO MainThread:2573666 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-18 21:03:15,369 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: shutdown +2025-01-18 21:03:15,369 INFO HandlerThread:2573666 [handler.py:finish():882] shutting down handler +2025-01-18 21:03:16,300 INFO WriterThread:2573666 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/run-7sdjk1lj.wandb +2025-01-18 21:03:16,368 INFO SenderThread:2573666 [sender.py:finish():1615] shutting down sender +2025-01-18 21:03:16,368 INFO SenderThread:2573666 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 21:03:16,368 INFO SenderThread:2573666 [file_pusher.py:join():175] waiting for file pusher diff --git a/7sdjk1lj/wandb/debug.log b/7sdjk1lj/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f3e34b33e2e0768549add31a05ca468316aaa1fb --- /dev/null +++ b/7sdjk1lj/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_setup.py:_flush():76] Configure stats pid to 2573483 +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/settings +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/logs/debug.log +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/logs/debug-internal.log +2025-01-18 20:46:12,096 INFO MainThread:2573483 [wandb_init.py:init():569] calling init triggers +2025-01-18 20:46:12,096 INFO MainThread:2573483 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_1', 'wandb_notes': '3465_1_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj'} +2025-01-18 20:46:12,096 INFO MainThread:2573483 [wandb_init.py:init():619] starting backend +2025-01-18 20:46:12,096 INFO MainThread:2573483 [wandb_init.py:init():623] setting up manager +2025-01-18 20:46:12,102 INFO MainThread:2573483 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-18 20:46:12,104 INFO MainThread:2573483 [wandb_init.py:init():631] backend started and connected +2025-01-18 20:46:12,115 INFO MainThread:2573483 [wandb_init.py:init():720] updated telemetry +2025-01-18 20:46:12,121 INFO MainThread:2573483 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-18 20:46:12,412 INFO MainThread:2573483 [wandb_run.py:_on_init():2435] communicating current version +2025-01-18 20:46:12,452 INFO MainThread:2573483 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-18 20:46:12,452 INFO MainThread:2573483 [wandb_init.py:init():804] starting run threads in backend +2025-01-18 20:46:12,854 INFO MainThread:2573483 [wandb_run.py:_console_start():2413] atexit reg +2025-01-18 20:46:12,854 INFO MainThread:2573483 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-18 20:46:12,854 INFO MainThread:2573483 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-18 20:46:12,854 INFO MainThread:2573483 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-18 20:46:12,855 INFO MainThread:2573483 [wandb_init.py:init():847] run started, returning control to user process +2025-01-18 20:46:12,856 INFO MainThread:2573483 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/tb, True +2025-01-18 20:46:30,268 INFO MainThread:2573483 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-18 21:03:16,369 WARNING MsgRouterThr:2573483 [router.py:message_loop():77] message_loop has been closed diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/code/FISH/eval_robot.py b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/config.yaml b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b726dc724182ac6d1301ae19d64f5421553a2884 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/config.yaml @@ -0,0 +1,927 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '3465_1' +wandb_notes: + desc: null + value: 3465_1_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: null +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 20 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737251172 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/diff.patch b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..9ed68bc2010bf65a550d961933cb7c74e1f0a0f0 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/diff.patch @@ -0,0 +1,102 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0b377ae..44ba2f5 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -132,8 +132,18 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1000_0' # all + + # all books, 10/20 demos per book ++# crp_D+act history ++# wandb_run_id: '3456_0' ++# wandb_run_id: '3456_1' # seed 1 ++# wandb_run_id: '3456_2' # seed 2 ++ ++# all books, 06/20 demos per book ++# crp_D+act history ++# wandb_run_id: '3465_0' ++wandb_run_id: '3465_1' # seed 1 ++# wandb_run_id: '3465_2' # seed 2 ++ + # RGBD+mask+act history +-wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + +@@ -191,7 +201,7 @@ wandb_run_id: '1002_0' + true_action_history: false + wandb_notes: null + +-checkpoint_epoch: 12000 # for depth + all contact ++checkpoint_epoch: 7500 # for depth + all contact + load_residual_weight: false + + checkpoint_root_dir: '/home/${oc.env:USER}/fish_leon/FISH' +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..9509617 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,15 +167,21 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ # '3456_0', ++ # '3456_1', ++ # '3456_2', ++ '3465_0', ++ '3465_1', ++ '3465_2', + ] + +-checkpoint_epoch = 12000 ++checkpoint_epoch = 7500 + + # use subprocess to download the model checkpoints in parallel + process_list = [] +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/table/eval/success_rate_ci_20_626687eda62aee1b0486.table.json b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/table/eval/success_rate_ci_20_626687eda62aee1b0486.table.json new file mode 100644 index 0000000000000000000000000000000000000000..761057ef622e692c8f8278d0c0a64eb320b0ba4d --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/table/eval/success_rate_ci_20_626687eda62aee1b0486.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.8283824323951083, 1, 0.8437453358677973, 1, 0.8703026323769969, 1, 1, 1]]} \ No newline at end of file diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/table/eval/total_success_rate_ci_21_7c95301d486f1fc093c2.table.json b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/table/eval/total_success_rate_ci_21_7c95301d486f1fc093c2.table.json new file mode 100644 index 0000000000000000000000000000000000000000..8b8c0de7525f1b75b57985c1baa4babb9873e9e1 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/table/eval/total_success_rate_ci_21_7c95301d486f1fc093c2.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[20, 20, 1.0, 0.8052949606761695, 1, 0.8727810233085844, 1, 0.8682639151075073, 1, 0.894491813998002, 1]]} \ No newline at end of file diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/0_eval_0_8f82b238c966cdc59610.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/0_eval_0_8f82b238c966cdc59610.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..476cae50772f07b2683a4883bd6cd06a7f4df761 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/0_eval_0_8f82b238c966cdc59610.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f82b238c966cdc59610e3bdaee3cca83001ea6f8f2ae2621486284349d030ed +size 1157700 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/10_eval_10_e5b99072831fb49e316b.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/10_eval_10_e5b99072831fb49e316b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..334fc299d0fb224412afded1c9146105e08e9608 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/10_eval_10_e5b99072831fb49e316b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5b99072831fb49e316be3f9d7fd1ae3d8921ee855c67b0290c257e463075d6b +size 1175376 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/11_eval_11_c3bd4b9c68e23c764392.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/11_eval_11_c3bd4b9c68e23c764392.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3fe69c6b077c98e407a107eacd3ee7ec2b7366f7 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/11_eval_11_c3bd4b9c68e23c764392.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3bd4b9c68e23c76439281ef7fe4b994676a06ff11d5614162dfe57a200e1c74 +size 1166965 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/12_eval_12_941e17d058fd7b746278.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/12_eval_12_941e17d058fd7b746278.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..77f9752e08f1c663e469f0bded510e24b3443e84 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/12_eval_12_941e17d058fd7b746278.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:941e17d058fd7b7462780a3767dd17bcf47dbde226e47dc9793ba48f5b7c3175 +size 1149306 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/13_eval_13_cf7080346708b3da4f52.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/13_eval_13_cf7080346708b3da4f52.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f876cb7acd4edaec9fccef23dcab78db86a73399 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/13_eval_13_cf7080346708b3da4f52.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf7080346708b3da4f5233e2677e4e537d3592ee4f99554b58490a7e9b6046c6 +size 1390055 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/14_eval_14_38839f2b6468bf8a5f05.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/14_eval_14_38839f2b6468bf8a5f05.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a99f2db201d68ca5fc3e07d970081bef5026a730 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/14_eval_14_38839f2b6468bf8a5f05.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38839f2b6468bf8a5f0557352e3b2be35a5202d7efdbd4b66c629228c28a37e1 +size 1382357 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/15_eval_15_3c41565be23b9dc07e38.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/15_eval_15_3c41565be23b9dc07e38.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2a98d426ce88f4f63a520fac34c47e4291db88ce --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/15_eval_15_3c41565be23b9dc07e38.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c41565be23b9dc07e38899b948a293b5fcda9b91f612a04698cc84685ae7c92 +size 1454269 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/16_eval_16_92f238d9751dde8bacfd.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/16_eval_16_92f238d9751dde8bacfd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8e54e71390a9f8e8932b0c5e670b9ef436bab88b --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/16_eval_16_92f238d9751dde8bacfd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92f238d9751dde8bacfd5ba98d52c91f9609ffff8bb07c892e3c0146f4fa28e6 +size 1150117 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/17_eval_17_cff7eb6bb7d10755b32e.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/17_eval_17_cff7eb6bb7d10755b32e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..24d39f9b1c74d2d74331d491336420eb0167031e --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/17_eval_17_cff7eb6bb7d10755b32e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cff7eb6bb7d10755b32eccdb52d1b797202d44a3cbc096866b4b9408d0eb6deb +size 1398146 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/18_eval_18_831456511316b3120131.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/18_eval_18_831456511316b3120131.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6ef0b60a5d21f2a0fd323c87cb2f7252612bf4c7 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/18_eval_18_831456511316b3120131.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:831456511316b3120131a6f259078ab1d77677226790bfa29dc32863b69e1a2f +size 1503368 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/19_eval_19_cc714c15f81700a6ca67.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/19_eval_19_cc714c15f81700a6ca67.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..38f2275245b26b9ea2266fa089d27b217cbaddf2 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/19_eval_19_cc714c15f81700a6ca67.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc714c15f81700a6ca671303f08983a4f63b247b5594bbeb385a946ba8ce35f4 +size 1573555 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/1_eval_1_f99205433ef4972a7f4b.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/1_eval_1_f99205433ef4972a7f4b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..74950544ed48dcd052fd4ad3d24daeb6551b8453 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/1_eval_1_f99205433ef4972a7f4b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f99205433ef4972a7f4bd939ee369818817757878cf57aa2f16a34c61cbc4e26 +size 1159170 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/2_eval_2_f1446b1ad582539ee197.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/2_eval_2_f1446b1ad582539ee197.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..62c744f7b9d789de990e66c281ea21403ccc289a --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/2_eval_2_f1446b1ad582539ee197.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1446b1ad582539ee197fad75893137396a64e85921942e998932e912d659ea4 +size 1159166 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/3_eval_3_b0e12a1c923453fa2db6.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/3_eval_3_b0e12a1c923453fa2db6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e37db5ea0d9d38ce53ddb5a60cb109004f927428 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/3_eval_3_b0e12a1c923453fa2db6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0e12a1c923453fa2db69a1f45ec1a121ab7b26d16dcb037170480cfcfeacbaf +size 1146532 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/4_eval_4_c2149e115233e8c46adb.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/4_eval_4_c2149e115233e8c46adb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e14956217b05c2947c0a82970aa8b1877ceb5f30 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/4_eval_4_c2149e115233e8c46adb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2149e115233e8c46adb595ad1e67e57c08904c4eb9560b94d48b032a3e82cd0 +size 1200313 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/5_eval_5_b6ffac2c1558879bc8f9.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/5_eval_5_b6ffac2c1558879bc8f9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2f3b923d47557524466693af54b125e3f17e7b2a --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/5_eval_5_b6ffac2c1558879bc8f9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6ffac2c1558879bc8f9bbb5265883a26b68096607d868548218d1880d433043 +size 1208719 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/6_eval_6_b37677ea4de36477338d.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/6_eval_6_b37677ea4de36477338d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..16d3dca161940dccd70a2b86f82d98180a7aba51 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/6_eval_6_b37677ea4de36477338d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b37677ea4de36477338da47977149554a61665dc80dd06131c39be59fede619b +size 1145126 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/7_eval_7_09f63e9126c49c6ebd71.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/7_eval_7_09f63e9126c49c6ebd71.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..41d12668d5d2b6f403459711751997dd9de5a16d --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/7_eval_7_09f63e9126c49c6ebd71.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09f63e9126c49c6ebd71be8e8336e3403d6ed934dc7cb8186565e6aa14214662 +size 1238849 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/8_eval_8_9db24d2dbee878841c0a.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/8_eval_8_9db24d2dbee878841c0a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f29b3e32f3e4346972896809b15529485c45311d --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/8_eval_8_9db24d2dbee878841c0a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9db24d2dbee878841c0adadd92b979ed83f33bd62249f56b1630a875df739652 +size 1273617 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/9_eval_9_bdaa1599f7ed02f8d4c8.mp4 b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/9_eval_9_bdaa1599f7ed02f8d4c8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68fa2a91fa1eb882dee86e6b9bbe0ce7f550e055 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/9_eval_9_bdaa1599f7ed02f8d4c8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdaa1599f7ed02f8d4c847b92de72ce6ff375db54a425253f139d1cc8fb440a3 +size 1275582 diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a1a306d494caa473222c99795d3242ea2fc6388e --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log @@ -0,0 +1,1088 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737251190.269291]: resetting environment +[INFO] [1737251190.272551]: cleared current plan +[INFO] [1737251190.273611]: moving to home +[INFO] [1737251191.275756]: reached home +[INFO] [1737251191.277471]: reset action history +[INFO] [1737251193.986897]: environment reset +Starting episode 0 +[INFO] [1737251193.989436]: resetting environment +[INFO] [1737251193.993272]: cleared current plan +[INFO] [1737251193.994179]: moving to home +[INFO] [1737251194.996549]: reached home +[INFO] [1737251194.997872]: reset action history +[INFO] [1737251197.709492]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1737251210.012401]: resetting environment +[INFO] [1737251210.015765]: cleared current plan +[INFO] [1737251210.016557]: moving to home +[INFO] [1737251211.019223]: reached home +[INFO] [1737251211.020676]: reset action history +[INFO] [1737251213.727727]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_0_2025-01-18-20-46-53.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:37497/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2573795] +process[print_text-2]: started with pid [2573819] +process[rosbag_record-3]: started with pid [2573820] +started bagging! +[INFO] [1737251240.714359]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1737251241.451170]: Stored episode 1. +[INFO] [1737251241.452019]: Saving video... +[INFO] [1737251241.643495]: Video saved! +Starting episode 1 +[INFO] [1737251241.662063]: resetting environment +[INFO] [1737251241.666362]: cleared current plan +[INFO] [1737251241.666624]: moving to home +[INFO] [1737251245.168770]: reached home +[INFO] [1737251245.169259]: reset action history +[INFO] [1737251247.876884]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737251256.849055]: resetting environment +[INFO] [1737251256.851193]: cleared current plan +[INFO] [1737251256.851386]: moving to home +[INFO] [1737251257.852972]: reached home +[INFO] [1737251257.853642]: reset action history +[INFO] [1737251260.560787]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_1_2025-01-18-20-47-40.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:46611/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2573938] +process[print_text-5]: started with pid [2573939] +process[rosbag_record-6]: started with pid [2573940] +started bagging! +[INFO] [1737251287.508035]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1737251288.274915]: Stored episode 2. +[INFO] [1737251288.275987]: Saving video... +[INFO] [1737251288.619569]: Video saved! +Starting episode 2 +[INFO] [1737251288.651451]: resetting environment +[INFO] [1737251288.660950]: cleared current plan +[INFO] [1737251288.661196]: moving to home +[INFO] [1737251292.163733]: reached home +[INFO] [1737251292.166378]: reset action history +[INFO] [1737251294.883695]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737251300.399995]: resetting environment +[INFO] [1737251300.409719]: cleared current plan +[INFO] [1737251300.410384]: moving to home +[INFO] [1737251301.412057]: reached home +[INFO] [1737251301.412827]: reset action history +[INFO] [1737251304.123256]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_2_2025-01-18-20-48-24.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:41445/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2574084] +process[print_text-8]: started with pid [2574085] +process[rosbag_record-9]: started with pid [2574086] +started bagging! +For topic gripper_width: timestamp difference is -101750738 for nearest: 1737251304502345675 - target: 1737251304604096413 at idx 8 +For topic gripper_width: timestamp difference is -101695131 for nearest: 1737251304702240397 - target: 1737251304803935528 at idx 9 +For topic gripper_width: timestamp difference is 106494177 for nearest: 1737251315102237452 - target: 1737251314995743275 at idx 148 +[INFO] [1737251331.061217]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1737251331.844875]: Stored episode 3. +[INFO] [1737251331.848780]: Saving video... +[INFO] [1737251332.191331]: Video saved! +Starting episode 3 +[INFO] [1737251332.228367]: resetting environment +[INFO] [1737251332.241088]: cleared current plan +[INFO] [1737251332.241924]: moving to home +[WARN] [1737251332.208353]: Plan exhausted +[INFO] [1737251335.857427]: reached home +[INFO] [1737251335.861477]: reset action history +[INFO] [1737251338.632680]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737251340.090139]: resetting environment +[INFO] [1737251340.097339]: cleared current plan +[INFO] [1737251340.098168]: moving to home +[INFO] [1737251341.103537]: reached home +[INFO] [1737251341.114629]: reset action history +[INFO] [1737251343.842590]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_3_2025-01-18-20-49-03.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:34083/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2574241] +process[print_text-11]: started with pid [2574242] +process[rosbag_record-12]: started with pid [2574243] +started bagging! +For topic gripper_width: timestamp difference is 102781019 for nearest: 1737251353002235136 - target: 1737251352899454117 at idx 133 +For topic gripper_width: timestamp difference is 103179887 for nearest: 1737251353202237085 - target: 1737251353099057198 at idx 134 +For topic gripper_width: timestamp difference is 106087848 for nearest: 1737251354602259084 - target: 1737251354496171236 at idx 151 +For topic gripper_width: timestamp difference is -120161799 for nearest: 1737251364202267858 - target: 1737251364322429657 at idx 249 +[INFO] [1737251370.985724]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit[pub_text-10] killing on exit +[INFO] [1737251371.766610]: Stored episode 4. +[INFO] [1737251371.767025]: Saving video... +[INFO] [1737251372.359919]: Video saved! +Starting episode 4 +[INFO] [1737251372.407808]: resetting environment +[INFO] [1737251372.425585]: cleared current plan +[INFO] [1737251372.425990]: moving to home +[WARN] [1737251372.328297]: Plan exhausted +[WARN] [1737251372.368648]: Plan exhausted +[WARN] [1737251372.411418]: Plan exhausted +[INFO] [1737251375.938131]: reached home +[INFO] [1737251375.943984]: reset action history +[INFO] [1737251378.732471]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737251379.575325]: resetting environment +[INFO] [1737251379.597996]: cleared current plan +[INFO] [1737251379.599208]: moving to home +[INFO] [1737251380.603770]: reached home +[INFO] [1737251380.607859]: reset action history +[INFO] [1737251383.388686]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_4_2025-01-18-20-49-43.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:44189/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2574384] +process[print_text-14]: started with pid [2574385] +process[rosbag_record-15]: started with pid [2574386] +started bagging! +For topic EE_pose: timestamp difference is 212018704 for nearest: 1737251385321318602 - target: 1737251385109299898 at idx 0 +For topic gripper_width: timestamp difference is -116755407 for nearest: 1737251409902261336 - target: 1737251410019016743 at idx 249 +[INFO] [1737251411.476972]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1737251412.264301]: Stored episode 5. +[INFO] [1737251412.267986]: Saving video... +[INFO] [1737251412.486539]: Video saved! +Starting episode 5 +[INFO] [1737251412.523193]: resetting environment +[INFO] [1737251412.538969]: cleared current plan +[INFO] [1737251412.541375]: moving to home +[INFO] [1737251416.149018]: reached home +[INFO] [1737251416.161858]: reset action history +[INFO] [1737251419.002062]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737251422.079751]: resetting environment +[INFO] [1737251422.086975]: cleared current plan +[INFO] [1737251422.090113]: moving to home +[INFO] [1737251423.095082]: reached home +[INFO] [1737251423.103373]: reset action history +[INFO] [1737251425.894130]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_5_2025-01-18-20-50-25.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:33839/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-16]: started with pid [2574529] +process[print_text-17]: started with pid [2574530] +process[rosbag_record-18]: started with pid [2574531] +started bagging! +[INFO] [1737251454.314857]: Storing episode... +[rosbag_record-18] killing on exit +[print_text-17] killing on exit +[pub_text-16] killing on exit +[WARN] [1737251455.688707]: Plan exhausted +[WARN] [1737251455.728089]: Plan exhausted +[WARN] [1737251455.768347]: Plan exhausted +[WARN] [1737251455.807933]: Plan exhausted +[INFO] [1737251455.111131]: Stored episode 6. +[INFO] [1737251455.117034]: Saving video... +[INFO] [1737251455.782595]: Video saved! +Starting episode 6 +[INFO] [1737251455.820498]: resetting environment +[INFO] [1737251455.847199]: cleared current plan +[INFO] [1737251455.853713]: moving to home +[INFO] [1737251459.459841]: reached home +[INFO] [1737251459.467555]: reset action history +[INFO] [1737251462.331991]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737251465.607310]: resetting environment +[INFO] [1737251465.628456]: cleared current plan +[INFO] [1737251465.637686]: moving to home +[INFO] [1737251466.641438]: reached home +[INFO] [1737251466.648241]: reset action history +[INFO] [1737251469.427264]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_6_2025-01-18-20-51-09.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:33531/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-19]: started with pid [2574638] +process[print_text-20]: started with pid [2574639] +process[rosbag_record-21]: started with pid [2574663] +started bagging! +[INFO] [1737251496.403791]: Storing episode... +[rosbag_record-21] killing on exit +[print_text-20] killing on exit[pub_text-19] killing on exit +[INFO] [1737251497.209673]: Stored episode 7. +[INFO] [1737251497.210053]: Saving video... +[INFO] [1737251497.611197]: Video saved! +Starting episode 7 +[INFO] [1737251497.696296]: resetting environment +[INFO] [1737251497.716060]: cleared current plan +[INFO] [1737251497.717988]: moving to home +[INFO] [1737251501.333590]: reached home +[INFO] [1737251501.345197]: reset action history +[INFO] [1737251504.213293]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737251507.450476]: resetting environment +[INFO] [1737251507.474248]: cleared current plan +[INFO] [1737251507.476483]: moving to home +[INFO] [1737251508.484230]: reached home +[INFO] [1737251508.494637]: reset action history +[INFO] [1737251511.302236]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_7_2025-01-18-20-51-51.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:38743/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-22]: started with pid [2574807] +process[print_text-23]: started with pid [2574808] +process[rosbag_record-24]: started with pid [2574809] +started bagging! +For topic gripper_width: timestamp difference is -161437043 for nearest: 1737251529602235786 - target: 1737251529763672829 at idx 249 +For topic gripper_width: timestamp difference is 126968761 for nearest: 1737251540468901058 - target: 1737251540341932297 at idx 249 +[INFO] [1737251540.649844]: Storing episode... +[rosbag_record-24] killing on exit +[print_text-23] killing on exit +[pub_text-22] killing on exit +[INFO] [1737251541.435143]: Stored episode 8. +[INFO] [1737251541.452782]: Saving video... +[INFO] [1737251541.981601]: Video saved! +Starting episode 8 +[INFO] [1737251542.077237]: resetting environment +[INFO] [1737251542.111996]: cleared current plan +[INFO] [1737251542.120844]: moving to home +[WARN] [1737251542.008268]: Plan exhausted +[WARN] [1737251542.047761]: Plan exhausted +[WARN] [1737251542.089564]: Plan exhausted +[INFO] [1737251545.647334]: reached home +[INFO] [1737251545.659947]: reset action history +[INFO] [1737251548.580183]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737251549.918913]: resetting environment +[INFO] [1737251549.963295]: cleared current plan +[INFO] [1737251549.965523]: moving to home +[INFO] [1737251550.978836]: reached home +[INFO] [1737251550.991086]: reset action history +[INFO] [1737251553.855909]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_8_2025-01-18-20-52-33.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:37747/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-25]: started with pid [2574951] +process[print_text-26]: started with pid [2574952] +process[rosbag_record-27]: started with pid [2574953] +started bagging! +[INFO] [1737251584.264418]: Storing episode... +[rosbag_record-27] killing on exit +[print_text-26] killing on exit +[pub_text-25] killing on exit +[INFO] [1737251585.008679]: Stored episode 9. +[INFO] [1737251585.041959]: Saving video... +[INFO] [1737251585.352316]: Video saved! +Starting episode 9 +[INFO] [1737251585.477715]: resetting environment +[INFO] [1737251585.515256]: cleared current plan +[INFO] [1737251585.519978]: moving to home +[INFO] [1737251589.131143]: reached home +[INFO] [1737251589.134641]: reset action history +[INFO] [1737251592.063803]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737251596.181334]: resetting environment +[INFO] [1737251596.194270]: cleared current plan +[INFO] [1737251596.199767]: moving to home +[INFO] [1737251597.218519]: reached home +[INFO] [1737251597.221241]: reset action history +[INFO] [1737251600.087905]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_9_2025-01-18-20-53-20.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:43649/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-28]: started with pid [2575099] +process[print_text-29]: started with pid [2575100] +process[rosbag_record-30]: started with pid [2575101] +started bagging! +[INFO] [1737251630.006634]: Storing episode... +[rosbag_record-30] killing on exit +[print_text-29] killing on exit +[pub_text-28] killing on exit +[INFO] [1737251630.786481]: Stored episode 10. +[INFO] [1737251630.801404]: Saving video... +[WARN] [1737251631.369697]: Plan exhausted +[WARN] [1737251631.410462]: Plan exhausted +[WARN] [1737251631.448753]: Plan exhausted +[WARN] [1737251631.491368]: Plan exhausted +[INFO] [1737251631.360579]: Video saved! +Starting episode 10 +[INFO] [1737251631.462343]: resetting environment +[INFO] [1737251631.507137]: cleared current plan +[INFO] [1737251631.511592]: moving to home +[INFO] [1737251635.130358]: reached home +[INFO] [1737251635.138845]: reset action history +[INFO] [1737251638.068617]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737251643.782734]: resetting environment +[INFO] [1737251643.839460]: cleared current plan +[INFO] [1737251643.850505]: moving to home +[INFO] [1737251644.870802]: reached home +[INFO] [1737251644.881357]: reset action history +[INFO] [1737251647.815441]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_10_2025-01-18-20-54-07.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:41663/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-31]: started with pid [2575230] +process[print_text-32]: started with pid [2575231] +process[rosbag_record-33]: started with pid [2575232] +started bagging! +For topic gripper_width: timestamp difference is -122768496 for nearest: 1737251650202193643 - target: 1737251650324962139 at idx 41 +For topic gripper_width: timestamp difference is -118649767 for nearest: 1737251655402212097 - target: 1737251655520861864 at idx 106 +[INFO] [1737251674.981707]: Storing episode... +[rosbag_record-33] killing on exit +[print_text-32] killing on exit +[pub_text-31] killing on exit +[INFO] [1737251675.817374]: Stored episode 11. +[INFO] [1737251675.843927]: Saving video... +[INFO] [1737251676.391001]: Video saved! +Starting episode 11 +[INFO] [1737251676.525038]: resetting environment +[INFO] [1737251676.542372]: cleared current plan +[INFO] [1737251676.545185]: moving to home +[WARN] [1737251676.171114]: Plan exhausted +[WARN] [1737251676.210260]: Plan exhausted +[WARN] [1737251676.254759]: Plan exhausted +[WARN] [1737251676.291049]: Plan exhausted +[WARN] [1737251676.330261]: Plan exhausted +[WARN] [1737251676.372157]: Plan exhausted +[WARN] [1737251676.420418]: Plan exhausted +[WARN] [1737251676.448922]: Plan exhausted +[WARN] [1737251676.495047]: Plan exhausted +[WARN] [1737251676.528063]: Plan exhausted +[INFO] [1737251680.160822]: reached home +[INFO] [1737251680.196975]: reset action history +[INFO] [1737251683.208632]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737251690.201134]: resetting environment +[INFO] [1737251690.223247]: cleared current plan +[INFO] [1737251690.223891]: moving to home +[INFO] [1737251691.232266]: reached home +[INFO] [1737251691.244698]: reset action history +[INFO] [1737251694.209538]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_11_2025-01-18-20-54-54.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:41717/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-34]: started with pid [2575375] +process[print_text-35]: started with pid [2575376] +process[rosbag_record-36]: started with pid [2575377] +started bagging! +For topic gripper_width: timestamp difference is -113353327 for nearest: 1737251703402193949 - target: 1737251703515547276 at idx 115 +For topic gripper_width: timestamp difference is -120540880 for nearest: 1737251707002237536 - target: 1737251707122778416 at idx 154 +For topic gripper_width: timestamp difference is -122553034 for nearest: 1737251707202234106 - target: 1737251707324787140 at idx 155 +For topic gripper_width: timestamp difference is -116368259 for nearest: 1737251710202235495 - target: 1737251710318603754 at idx 194 +For topic gripper_width: timestamp difference is -113892539 for nearest: 1737251711402241723 - target: 1737251711516134262 at idx 208 +For topic gripper_width: timestamp difference is -111075122 for nearest: 1737251712802231830 - target: 1737251712913306952 at idx 228 +For topic gripper_width: timestamp difference is -107396086 for nearest: 1737251714602231304 - target: 1737251714709627390 at idx 247 +For topic gripper_width: timestamp difference is -100401113 for nearest: 1737251718002264073 - target: 1737251718102665186 at idx 249 +[INFO] [1737251721.371708]: Storing episode... +[pub_text-34] killing on exit +[print_text-35] killing on exit[rosbag_record-36] killing on exit +[INFO] [1737251722.208839]: Stored episode 12. +[INFO] [1737251722.217558]: Saving video... +[INFO] [1737251722.541232]: Video saved! +Starting episode 12 +[INFO] [1737251722.675787]: resetting environment +[INFO] [1737251722.722494]: cleared current plan +[INFO] [1737251722.724558]: moving to home +[INFO] [1737251726.344261]: reached home +[INFO] [1737251726.353732]: reset action history +[INFO] [1737251729.362750]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737251730.842267]: resetting environment +[INFO] [1737251730.909954]: cleared current plan +[INFO] [1737251730.912592]: moving to home +[INFO] [1737251731.928975]: reached home +[INFO] [1737251731.952312]: reset action history +[INFO] [1737251734.950011]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_12_2025-01-18-20-55-34.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:37139/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-37]: started with pid [2575495] +process[print_text-38]: started with pid [2575496] +process[rosbag_record-39]: started with pid [2575520] +started bagging! +For topic gripper_width: timestamp difference is 100320092 for nearest: 1737251760368855468 - target: 1737251760268535376 at idx 249 +For topic gripper_width: timestamp difference is 101983140 for nearest: 1737251761168899844 - target: 1737251761066916704 at idx 249 +[INFO] [1737251761.966210]: Storing episode... +[print_text-38] killing on exit +[pub_text-37] killing on exit +[rosbag_record-39] killing on exit +[INFO] [1737251762.817033]: Stored episode 13. +[INFO] [1737251762.821409]: Saving video... +[WARN] [1737251763.287992]: Plan exhausted +[WARN] [1737251763.329456]: Plan exhausted +[INFO] [1737251763.176732]: Video saved! +Starting episode 13 +[INFO] [1737251763.313764]: resetting environment +[INFO] [1737251763.365234]: cleared current plan +[INFO] [1737251763.372039]: moving to home +[INFO] [1737251766.992460]: reached home +[INFO] [1737251766.996644]: reset action history +[INFO] [1737251770.060751]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737251774.162348]: resetting environment +[INFO] [1737251774.251624]: cleared current plan +[INFO] [1737251774.282375]: moving to home +[INFO] [1737251775.292429]: reached home +[INFO] [1737251775.327085]: reset action history +[INFO] [1737251778.380832]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_13_2025-01-18-20-56-18.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:37915/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-40]: started with pid [2575663] +process[print_text-41]: started with pid [2575664] +process[rosbag_record-42]: started with pid [2575665] +started bagging! +For topic gripper_width: timestamp difference is 100642962 for nearest: 1737251791868909878 - target: 1737251791768266916 at idx 194 +For topic gripper_width: timestamp difference is 102129374 for nearest: 1737251792868856106 - target: 1737251792766726732 at idx 211 +For topic gripper_width: timestamp difference is -104377744 for nearest: 1737251797102235320 - target: 1737251797206613064 at idx 249 +For topic gripper_width: timestamp difference is -104671659 for nearest: 1737251802902235804 - target: 1737251803006907463 at idx 249 +For topic gripper_width: timestamp difference is -104047277 for nearest: 1737251807102255843 - target: 1737251807206303120 at idx 249 +[INFO] [1737251811.872237]: Storing episode... +[print_text-41] killing on exit +[pub_text-40] killing on exit +[rosbag_record-42] killing on exit +[INFO] [1737251812.749187]: Stored episode 14. +[INFO] [1737251812.754362]: Saving video... +[WARN] [1737251813.207765]: Plan exhausted +[WARN] [1737251813.255923]: Plan exhausted +[WARN] [1737251813.289769]: Plan exhausted +[WARN] [1737251813.330338]: Plan exhausted +[WARN] [1737251813.368288]: Plan exhausted +[WARN] [1737251813.409042]: Plan exhausted +[WARN] [1737251813.449239]: Plan exhausted +[WARN] [1737251813.487761]: Plan exhausted +[WARN] [1737251813.530990]: Plan exhausted +[WARN] [1737251813.569172]: Plan exhausted +[INFO] [1737251813.430974]: Video saved! +Starting episode 14 +[INFO] [1737251813.588385]: resetting environment +[INFO] [1737251813.604021]: cleared current plan +[INFO] [1737251813.604261]: moving to home +[INFO] [1737251817.118279]: reached home +[INFO] [1737251817.146680]: reset action history +[INFO] [1737251820.162029]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737251826.378895]: resetting environment +[INFO] [1737251826.432517]: cleared current plan +[INFO] [1737251826.439737]: moving to home +[INFO] [1737251827.457205]: reached home +[INFO] [1737251827.471943]: reset action history +[INFO] [1737251830.547283]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_14_2025-01-18-20-57-10.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:34583/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-43]: started with pid [2575794] +process[print_text-44]: started with pid [2575795] +process[rosbag_record-45]: started with pid [2575796] +started bagging! +[INFO] [1737251863.224389]: Storing episode... +[rosbag_record-45] killing on exit +[pub_text-43] killing on exit[print_text-44] killing on exit +[INFO] [1737251864.028575]: Stored episode 15. +[INFO] [1737251864.039664]: Saving video... +[INFO] [1737251864.472186]: Video saved! +Starting episode 15 +[INFO] [1737251864.628992]: resetting environment +[INFO] [1737251864.717349]: cleared current plan +[INFO] [1737251864.721344]: moving to home +[WARN] [1737251864.573623]: Plan exhausted +[WARN] [1737251864.608155]: Plan exhausted +[WARN] [1737251864.658872]: Plan exhausted +[WARN] [1737251864.689149]: Plan exhausted +[INFO] [1737251868.341406]: reached home +[INFO] [1737251868.346178]: reset action history +[INFO] [1737251871.438045]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737251880.512250]: resetting environment +[INFO] [1737251880.599516]: cleared current plan +[INFO] [1737251880.609880]: moving to home +[INFO] [1737251881.637067]: reached home +[INFO] [1737251881.654090]: reset action history +[INFO] [1737251884.726990]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_15_2025-01-18-20-58-04.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:37743/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-46]: started with pid [2575939] +process[print_text-47]: started with pid [2575940] +process[rosbag_record-48]: started with pid [2575941] +started bagging! +[INFO] [1737251919.311885]: Storing episode... +[rosbag_record-48] killing on exit +[print_text-47] killing on exit[pub_text-46] killing on exit +[INFO] [1737251920.166591]: Stored episode 16. +[INFO] [1737251920.184740]: Saving video... +[INFO] [1737251920.920368]: Video saved! +Starting episode 16 +[INFO] [1737251921.061174]: resetting environment +[INFO] [1737251921.112379]: cleared current plan +[INFO] [1737251921.117174]: moving to home +[WARN] [1737251920.647767]: Plan exhausted +[WARN] [1737251920.695081]: Plan exhausted +[WARN] [1737251920.727916]: Plan exhausted +[WARN] [1737251920.767774]: Plan exhausted +[WARN] [1737251920.810705]: Plan exhausted +[WARN] [1737251920.856859]: Plan exhausted +[WARN] [1737251920.890177]: Plan exhausted +[WARN] [1737251920.930869]: Plan exhausted +[WARN] [1737251920.967757]: Plan exhausted +[WARN] [1737251921.010343]: Plan exhausted +[WARN] [1737251921.067439]: Plan exhausted +[WARN] [1737251921.088746]: Plan exhausted +[INFO] [1737251924.732037]: reached home +[INFO] [1737251924.757389]: reset action history +[INFO] [1737251927.950246]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737251940.771683]: resetting environment +[INFO] [1737251940.910011]: cleared current plan +[INFO] [1737251940.936672]: moving to home +[INFO] [1737251941.953152]: reached home +[INFO] [1737251941.976146]: reset action history +[INFO] [1737251945.247328]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_16_2025-01-18-20-59-05.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:45967/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-49]: started with pid [2576065] +process[print_text-50]: started with pid [2576089] +process[rosbag_record-51]: started with pid [2576090] +started bagging! +For topic gripper_width: timestamp difference is -107717487 for nearest: 1737251958102231291 - target: 1737251958209948778 at idx 182 +[INFO] [1737251972.253898]: Storing episode... +[rosbag_record-51] killing on exit +[print_text-50] killing on exit +[pub_text-49] killing on exit +[INFO] [1737251973.134849]: Stored episode 17. +[INFO] [1737251973.152420]: Saving video... +[WARN] [1737251973.612367]: Plan exhausted +[WARN] [1737251973.648650]: Plan exhausted +[WARN] [1737251973.689010]: Plan exhausted +[WARN] [1737251973.730240]: Plan exhausted +[WARN] [1737251973.771986]: Plan exhausted +[WARN] [1737251973.817452]: Plan exhausted +[WARN] [1737251973.851397]: Plan exhausted +[WARN] [1737251973.887774]: Plan exhausted +[WARN] [1737251973.929669]: Plan exhausted +[WARN] [1737251973.971185]: Plan exhausted +[WARN] [1737251974.010501]: Plan exhausted +[WARN] [1737251974.053155]: Plan exhausted +[WARN] [1737251974.087865]: Plan exhausted +[INFO] [1737251973.829311]: Video saved! +Starting episode 17 +[INFO] [1737251974.029470]: resetting environment +[INFO] [1737251974.102384]: cleared current plan +[INFO] [1737251974.117600]: moving to home +[INFO] [1737251977.740421]: reached home +[INFO] [1737251977.762064]: reset action history +[INFO] [1737251980.948172]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737251986.568988]: resetting environment +[INFO] [1737251986.671996]: cleared current plan +[INFO] [1737251986.675704]: moving to home +[INFO] [1737251987.701063]: reached home +[INFO] [1737251987.712394]: reset action history +[INFO] [1737251990.764427]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_17_2025-01-18-20-59-50.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:44513/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-52]: started with pid [2576233] +process[print_text-53]: started with pid [2576234] +process[rosbag_record-54]: started with pid [2576235] +started bagging! +For topic gripper_width: timestamp difference is -237431798 for nearest: 1737252003702231374 - target: 1737252003939663172 at idx 178 +For topic gripper_width: timestamp difference is -229783938 for nearest: 1737252013302229240 - target: 1737252013532013178 at idx 249 +For topic gripper_width: timestamp difference is -228949759 for nearest: 1737252014302214172 - target: 1737252014531163931 at idx 249 +[INFO] [1737252024.607602]: Storing episode... +[rosbag_record-54] killing on exit +[print_text-53] killing on exit +[pub_text-52] killing on exit +[INFO] [1737252025.498727]: Stored episode 18. +[INFO] [1737252025.521075]: Saving video... +[INFO] [1737252025.803100]: Video saved! +Starting episode 18 +[INFO] [1737252025.923879]: resetting environment +[INFO] [1737252025.963764]: cleared current plan +[INFO] [1737252025.964981]: moving to home +[INFO] [1737252029.576787]: reached home +[INFO] [1737252029.614687]: reset action history +[INFO] [1737252032.833497]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737252037.891033]: resetting environment +[INFO] [1737252037.934507]: cleared current plan +[INFO] [1737252037.936662]: moving to home +[INFO] [1737252038.947343]: reached home +[INFO] [1737252038.968454]: reset action history +[INFO] [1737252042.092650]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_18_2025-01-18-21-00-42.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:39641/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-55]: started with pid [2576377] +process[print_text-56]: started with pid [2576378] +process[rosbag_record-57]: started with pid [2576379] +started bagging! +[INFO] [1737252078.731131]: Storing episode... +[rosbag_record-57] killing on exit +[pub_text-55] killing on exit +[print_text-56] killing on exit +[WARN] [1737252080.089610]: Plan exhausted +[WARN] [1737252080.134129]: Plan exhausted +[WARN] [1737252080.171798]: Plan exhausted +[WARN] [1737252080.228953]: Plan exhausted +[WARN] [1737252080.269378]: Plan exhausted +[WARN] [1737252080.298746]: Plan exhausted +[WARN] [1737252080.338219]: Plan exhausted +[WARN] [1737252080.406801]: Plan exhausted +[INFO] [1737252079.608497]: Stored episode 19. +[INFO] [1737252079.627043]: Saving video... +[INFO] [1737252080.214851]: Video saved! +Starting episode 19 +[INFO] [1737252080.467873]: resetting environment +[INFO] [1737252080.519992]: cleared current plan +[INFO] [1737252080.536621]: moving to home +[WARN] [1737252080.476797]: Plan exhausted +[WARN] [1737252080.493489]: Plan exhausted +[WARN] [1737252080.505460]: Plan exhausted +[INFO] [1737252084.167203]: reached home +[INFO] [1737252084.188622]: reset action history +[INFO] [1737252087.430500]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737252089.868533]: resetting environment +[INFO] [1737252089.990070]: cleared current plan +[INFO] [1737252090.022126]: moving to home +[INFO] [1737252091.054830]: reached home +[INFO] [1737252091.067521]: reset action history +[INFO] [1737252094.359474]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/episode_rosbags/episode_19_2025-01-18-21-01-34.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2573483.log +started roslaunch server http://158.130.50.37:42115/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-58]: started with pid [2576521] +process[print_text-59]: started with pid [2576522] +process[rosbag_record-60]: started with pid [2576523] +started bagging! +[INFO] [1737252132.152322]: Storing episode... +[rosbag_record-60] killing on exit +[print_text-59] killing on exit +[pub_text-58] killing on exit +[INFO] [1737252133.055334]: Stored episode 20. +[INFO] [1737252133.072693]: Saving video... +[WARN] [1737252133.492695]: Plan exhausted +[WARN] [1737252133.528076]: Plan exhausted +[WARN] [1737252133.569430]: Plan exhausted +[WARN] [1737252133.615357]: Plan exhausted +[WARN] [1737252133.661453]: Plan exhausted +[WARN] [1737252133.687883]: Plan exhausted +[WARN] [1737252133.729552]: Plan exhausted +[WARN] [1737252133.767789]: Plan exhausted +[WARN] [1737252133.824133]: Plan exhausted +[WARN] [1737252133.847794]: Plan exhausted +[WARN] [1737252133.888204]: Plan exhausted +[WARN] [1737252133.935602]: Plan exhausted +[WARN] [1737252133.990664]: Plan exhausted +[WARN] [1737252134.031595]: Plan exhausted +[INFO] [1737252133.796526]: Video saved! +[INFO] [1737252133.970925]: resetting environment +[INFO] [1737252134.044111]: cleared current plan +[INFO] [1737252134.047301]: moving to home +[INFO] [1737252137.673207]: reached home +[INFO] [1737252137.696199]: reset action history +[INFO] [1737252141.094974]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/requirements.txt b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch new file mode 100644 index 0000000000000000000000000000000000000000..155e5035dfd929cfac1e198ea669aef65e872cef --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch @@ -0,0 +1,119 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..44ba2f5 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 5 ++num_eval: 20 + + debug_timestamps: False + open_loop: False +@@ -132,15 +132,25 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1000_0' # all + + # all books, 10/20 demos per book ++# crp_D+act history ++# wandb_run_id: '3456_0' ++# wandb_run_id: '3456_1' # seed 1 ++# wandb_run_id: '3456_2' # seed 2 ++ ++# all books, 06/20 demos per book ++# crp_D+act history ++# wandb_run_id: '3465_0' ++wandb_run_id: '3465_1' # seed 1 ++# wandb_run_id: '3465_2' # seed 2 ++ + # RGBD+mask+act history +-# wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history +@@ -191,7 +201,7 @@ wandb_run_id: '1017_0' # dataset shuffle seed 1 + true_action_history: false + wandb_notes: null + +-checkpoint_epoch: 12000 # for depth + all contact ++checkpoint_epoch: 7500 # for depth + all contact + load_residual_weight: false + + checkpoint_root_dir: '/home/${oc.env:USER}/fish_leon/FISH' +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..9509617 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,15 +167,21 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ # '3456_0', ++ # '3456_1', ++ # '3456_2', ++ '3465_0', ++ '3465_1', ++ '3465_2', + ] + +-checkpoint_epoch = 12000 ++checkpoint_epoch = 7500 + + # use subprocess to download the model checkpoints in parallel + process_list = [] +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-metadata.json b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..19ae9ba813675c986ccdfe420a1514bcec594690 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-19T01:46:12.475557", + "startedAt": "2025-01-19T01:46:12.087278", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "ec42957dc10d08dbd8b134558fcd8f9e470ccc31" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3380.134625, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4563.213, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 631.7242088317871 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ba0eb2059e196ffe0cd8feb1de406d0d1483d751 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "8f82b238c966cdc59610e3bdaee3cca83001ea6f8f2ae2621486284349d030ed", "size": 1157700, "path": "media/videos/eval/0_eval_0_8f82b238c966cdc59610.mp4"}, "global_step": 2658, "_timestamp": 1737252148.6315653, "_runtime": 976.5272092819214, "_step": 41, "eval/1_eval": {"_type": "video-file", "sha256": "f99205433ef4972a7f4bd939ee369818817757878cf57aa2f16a34c61cbc4e26", "size": 1159170, "path": "media/videos/eval/1_eval_1_f99205433ef4972a7f4b.mp4"}, "eval/num_success": 20.0, "episode": 20.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.8283824324607849, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.8437453508377075, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.8703026175498962, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "f1446b1ad582539ee197fad75893137396a64e85921942e998932e912d659ea4", "size": 1159166, "path": "media/videos/eval/2_eval_2_f1446b1ad582539ee197.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "b0e12a1c923453fa2db69a1f45ec1a121ab7b26d16dcb037170480cfcfeacbaf", "size": 1146532, "path": "media/videos/eval/3_eval_3_b0e12a1c923453fa2db6.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "c2149e115233e8c46adb595ad1e67e57c08904c4eb9560b94d48b032a3e82cd0", "size": 1200313, "path": "media/videos/eval/4_eval_4_c2149e115233e8c46adb.mp4"}, "eval/5_eval": {"_type": "video-file", "sha256": "b6ffac2c1558879bc8f9bbb5265883a26b68096607d868548218d1880d433043", "size": 1208719, "path": "media/videos/eval/5_eval_5_b6ffac2c1558879bc8f9.mp4"}, "eval/6_eval": {"_type": "video-file", "sha256": "b37677ea4de36477338da47977149554a61665dc80dd06131c39be59fede619b", "size": 1145126, "path": "media/videos/eval/6_eval_6_b37677ea4de36477338d.mp4"}, "eval/7_eval": {"_type": "video-file", "sha256": "09f63e9126c49c6ebd71be8e8336e3403d6ed934dc7cb8186565e6aa14214662", "size": 1238849, "path": "media/videos/eval/7_eval_7_09f63e9126c49c6ebd71.mp4"}, "eval/8_eval": {"_type": "video-file", "sha256": "9db24d2dbee878841c0adadd92b979ed83f33bd62249f56b1630a875df739652", "size": 1273617, "path": "media/videos/eval/8_eval_8_9db24d2dbee878841c0a.mp4"}, "eval/9_eval": {"_type": "video-file", "sha256": "bdaa1599f7ed02f8d4c847b92de72ce6ff375db54a425253f139d1cc8fb440a3", "size": 1275582, "path": "media/videos/eval/9_eval_9_bdaa1599f7ed02f8d4c8.mp4"}, "eval/10_eval": {"_type": "video-file", "sha256": "e5b99072831fb49e316be3f9d7fd1ae3d8921ee855c67b0290c257e463075d6b", "size": 1175376, "path": "media/videos/eval/10_eval_10_e5b99072831fb49e316b.mp4"}, "eval/11_eval": {"_type": "video-file", "sha256": "c3bd4b9c68e23c76439281ef7fe4b994676a06ff11d5614162dfe57a200e1c74", "size": 1166965, "path": "media/videos/eval/11_eval_11_c3bd4b9c68e23c764392.mp4"}, "eval/12_eval": {"_type": "video-file", "sha256": "941e17d058fd7b7462780a3767dd17bcf47dbde226e47dc9793ba48f5b7c3175", "size": 1149306, "path": "media/videos/eval/12_eval_12_941e17d058fd7b746278.mp4"}, "eval/13_eval": {"_type": "video-file", "sha256": "cf7080346708b3da4f5233e2677e4e537d3592ee4f99554b58490a7e9b6046c6", "size": 1390055, "path": "media/videos/eval/13_eval_13_cf7080346708b3da4f52.mp4"}, "eval/14_eval": {"_type": "video-file", "sha256": "38839f2b6468bf8a5f0557352e3b2be35a5202d7efdbd4b66c629228c28a37e1", "size": 1382357, "path": "media/videos/eval/14_eval_14_38839f2b6468bf8a5f05.mp4"}, "eval/15_eval": {"_type": "video-file", "sha256": "3c41565be23b9dc07e38899b948a293b5fcda9b91f612a04698cc84685ae7c92", "size": 1454269, "path": "media/videos/eval/15_eval_15_3c41565be23b9dc07e38.mp4"}, "eval/16_eval": {"_type": "video-file", "sha256": "92f238d9751dde8bacfd5ba98d52c91f9609ffff8bb07c892e3c0146f4fa28e6", "size": 1150117, "path": "media/videos/eval/16_eval_16_92f238d9751dde8bacfd.mp4"}, "eval/17_eval": {"_type": "video-file", "sha256": "cff7eb6bb7d10755b32eccdb52d1b797202d44a3cbc096866b4b9408d0eb6deb", "size": 1398146, "path": "media/videos/eval/17_eval_17_cff7eb6bb7d10755b32e.mp4"}, "eval/18_eval": {"_type": "video-file", "sha256": "831456511316b3120131a6f259078ab1d77677226790bfa29dc32863b69e1a2f", "size": 1503368, "path": "media/videos/eval/18_eval_18_831456511316b3120131.mp4"}, "eval/19_eval": {"_type": "video-file", "sha256": "cc714c15f81700a6ca671303f08983a4f63b247b5594bbeb385a946ba8ce35f4", "size": 1573555, "path": "media/videos/eval/19_eval_19_cc714c15f81700a6ca67.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "626687eda62aee1b048625eeaf932e17a6a6b800c6da18ff3cbb0c340b3e96b3", "size": 313, "artifact_path": "wandb-client-artifact://i2n8z6tuiy05zzjiy99975llrwkbz04hqdskunq5ay1efmp7a4n062216r3dtkexx8c4w3uo0njcienlnetxp14c6a3azqykcyackxlf1iy0k2gi6d3bbu4iugedhn5l/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://0mjsuudfscf1fmw4fxky06zr3jp1d260eescy5otvdkcfxac5egsfkwbfh3k99wt93bgu62z145zv4by4ra0j92t260amgbykfzlxkranmbzv1269e23ywbzb6o7ebut:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_20_626687eda62aee1b0486.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 20, "total_num_episodes": 20, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.8052949606761695, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.8727810233085844, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.8682639151075073, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 0.894491813998002, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "7c95301d486f1fc093c2b159f33ce63d82fc43d7ca4c22a98ba603061f726786", "size": 436, "artifact_path": "wandb-client-artifact://nvz9r5jtnrinwfn4jvk1q0t7wk9bxuibcpxtoufdijrmp0rf425norco9t1zierdq5hj5vkozfmc1i0hhn27d8x25tr4xe0oj7ktkqxg72bgzgoge03b6ol9hzeyegzc/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ok15pcer64i3o10kjfu41lkr8s01jvzghwy3zk99dtl5edtho3d7pr99yrc2hzqklz7oldsvbyh0nxqaq7l31cu8rfhkxrn5di9p968856i2du9s3x1l7c5s7hc2wh45:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_21_7c95301d486f1fc093c2.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 1011}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.805294930934906, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.8727810382843018, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.8682639002799988, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 0.8944917917251587, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/logs/debug-internal.log b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b1d7680372f012b87ede6700e8ce52ac9315cb77 --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/logs/debug-internal.log @@ -0,0 +1,2179 @@ +2025-01-18 20:46:12,103 INFO StreamThr :2573666 [internal.py:wandb_internal():85] W&B internal server running at pid: 2573666, started at: 2025-01-18 20:46:12.102899 +2025-01-18 20:46:12,104 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status +2025-01-18 20:46:12,115 INFO WriterThread:2573666 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/run-7sdjk1lj.wandb +2025-01-18 20:46:12,115 DEBUG SenderThread:2573666 [sender.py:send():379] send: header +2025-01-18 20:46:12,123 DEBUG SenderThread:2573666 [sender.py:send():379] send: run +2025-01-18 20:46:12,392 INFO SenderThread:2573666 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files +2025-01-18 20:46:12,392 INFO SenderThread:2573666 [sender.py:_start_run_threads():1188] run started: 7sdjk1lj with start time 1737251172.104356 +2025-01-18 20:46:12,412 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: check_version +2025-01-18 20:46:12,413 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: check_version +2025-01-18 20:46:12,457 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: run_start +2025-01-18 20:46:12,462 DEBUG HandlerThread:2573666 [system_info.py:__init__():26] System info init +2025-01-18 20:46:12,462 DEBUG HandlerThread:2573666 [system_info.py:__init__():41] System info init done +2025-01-18 20:46:12,462 INFO HandlerThread:2573666 [system_monitor.py:start():194] Starting system monitor +2025-01-18 20:46:12,462 INFO SystemMonitor:2573666 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-18 20:46:12,462 INFO HandlerThread:2573666 [system_monitor.py:probe():214] Collecting system info +2025-01-18 20:46:12,462 INFO SystemMonitor:2573666 [interfaces.py:start():188] Started cpu monitoring +2025-01-18 20:46:12,463 INFO SystemMonitor:2573666 [interfaces.py:start():188] Started disk monitoring +2025-01-18 20:46:12,463 INFO SystemMonitor:2573666 [interfaces.py:start():188] Started gpu monitoring +2025-01-18 20:46:12,464 INFO SystemMonitor:2573666 [interfaces.py:start():188] Started memory monitoring +2025-01-18 20:46:12,465 INFO SystemMonitor:2573666 [interfaces.py:start():188] Started network monitoring +2025-01-18 20:46:12,475 DEBUG HandlerThread:2573666 [system_info.py:probe():152] Probing system +2025-01-18 20:46:12,477 DEBUG HandlerThread:2573666 [system_info.py:_probe_git():137] Probing git +2025-01-18 20:46:12,481 DEBUG HandlerThread:2573666 [system_info.py:_probe_git():145] Probing git done +2025-01-18 20:46:12,481 DEBUG HandlerThread:2573666 [system_info.py:probe():200] Probing system done +2025-01-18 20:46:12,481 DEBUG HandlerThread:2573666 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-19T01:46:12.475557', 'startedAt': '2025-01-19T01:46:12.087278', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3380.134625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4563.213, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 631.7242088317871}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-18 20:46:12,481 INFO HandlerThread:2573666 [system_monitor.py:probe():224] Finished collecting system info +2025-01-18 20:46:12,481 INFO HandlerThread:2573666 [system_monitor.py:probe():227] Publishing system info +2025-01-18 20:46:12,481 DEBUG HandlerThread:2573666 [system_info.py:_save_code():44] Saving code +2025-01-18 20:46:12,487 DEBUG HandlerThread:2573666 [system_info.py:_save_code():67] Saving code done +2025-01-18 20:46:12,487 DEBUG HandlerThread:2573666 [system_info.py:_save_patches():84] Saving git patches +2025-01-18 20:46:12,586 DEBUG HandlerThread:2573666 [system_info.py:_save_patches():126] Saving git patches done +2025-01-18 20:46:12,587 INFO HandlerThread:2573666 [system_monitor.py:probe():229] Finished publishing system info +2025-01-18 20:46:12,590 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:46:12,590 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-18 20:46:12,590 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-18 20:46:12,591 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-18 20:46:12,591 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-18 20:46:12,853 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: python_packages +2025-01-18 20:46:12,853 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: python_packages +2025-01-18 20:46:12,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:46:12,854 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:46:12,867 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/8at10yt1-wandb-metadata.json +2025-01-18 20:46:12,870 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/wafsayyp-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 20:46:12,872 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/7dvjyqlt-code/FISH/eval_robot.py +2025-01-18 20:46:12,889 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/tm9i5og1-diff.patch +2025-01-18 20:46:12,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:12,897 INFO HandlerThread:2573666 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/tb" + save: true +} +control { + relay_id: "ffe869506cfe4ddda48be41fbc39527d" +} +_info { + stream_id: "7sdjk1lj" +} + +2025-01-18 20:46:12,897 DEBUG HandlerThread:2573666 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-18 20:46:12,956 DEBUG SenderThread:2573666 [sender.py:send():379] send: telemetry +2025-01-18 20:46:12,956 DEBUG SenderThread:2573666 [sender.py:send():379] send: tbrecord +2025-01-18 20:46:12,958 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:46:12,958 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 with policy live +2025-01-18 20:46:13,151 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/67lsmr8t-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:46:13,394 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/requirements.txt +2025-01-18 20:46:13,394 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/code/FISH/eval_robot.py +2025-01-18 20:46:13,394 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-metadata.json +2025-01-18 20:46:13,394 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 20:46:13,394 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:46:13,394 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:13,395 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/diff.patch +2025-01-18 20:46:13,395 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/code +2025-01-18 20:46:13,395 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/code/FISH +2025-01-18 20:46:13,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:14,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:15,393 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:15,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:16,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:17,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:46:17,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:18,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:19,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:20,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:21,856 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:22,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:46:22,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:23,856 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:24,856 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:25,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:26,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:27,853 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:46:27,854 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:46:27,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:27,942 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:46:28,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:29,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:30,269 DEBUG SenderThread:2573666 [sender.py:send():379] send: config +2025-01-18 20:46:30,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:31,397 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:31,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:32,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:33,398 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:33,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:46:33,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:34,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:35,399 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:35,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:36,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:37,400 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:37,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:38,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:39,400 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:39,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:46:39,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:40,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:41,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:42,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:46:42,854 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:46:42,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:43,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:44,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:44,978 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:46:45,402 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/config.yaml +2025-01-18 20:46:45,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:46,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:47,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:48,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:49,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:50,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:51,019 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:46:51,404 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:51,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:52,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:53,404 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:53,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:54,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:55,405 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:46:55,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:56,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:57,007 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:46:57,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:46:57,854 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:46:57,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:58,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:46:59,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:00,862 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:01,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:02,011 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:02,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:03,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:04,862 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:05,862 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:06,862 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:07,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:07,862 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:08,863 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:09,863 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:10,863 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:11,863 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:12,465 DEBUG SystemMonitor:2573666 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-18 20:47:12,465 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:47:12,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:47:12,854 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:47:12,901 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:12,954 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:13,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:14,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:15,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:16,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:17,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:18,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:18,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:19,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:20,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:21,661 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:47:21,661 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:47:21,661 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_8f82b238c966cdc59610.mp4 with policy now +2025-01-18 20:47:21,662 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:47:21,663 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:47:21,663 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:47:21,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:21,912 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/z47dc6lj-media/videos/eval/0_eval_0_8f82b238c966cdc59610.mp4 +2025-01-18 20:47:22,413 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:47:22,414 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/0_eval_0_8f82b238c966cdc59610.mp4 +2025-01-18 20:47:22,414 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos +2025-01-18 20:47:22,414 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:47:22,414 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media +2025-01-18 20:47:22,715 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/el3cbdz8-media/videos/eval/0_eval_0_8f82b238c966cdc59610.mp4 +2025-01-18 20:47:22,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:23,414 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:47:23,865 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:24,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:24,865 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:25,865 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:26,865 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:27,416 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:47:27,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:47:27,854 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:47:27,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:28,865 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:29,416 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:47:29,865 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:29,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:30,865 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:31,866 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:32,866 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:33,866 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:34,866 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:35,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:35,866 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:36,885 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:37,419 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:47:37,670 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/f8f0zrm2-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:47:37,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:38,867 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:39,420 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:47:39,867 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:40,867 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:41,789 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:41,867 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:42,468 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:47:42,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:47:42,854 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:47:42,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:43,422 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:47:43,867 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:44,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:45,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:46,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:46,929 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:47,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:48,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:49,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:50,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:51,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:52,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:52,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:53,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:54,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:55,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:56,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:57,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:47:57,854 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:47:57,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:58,010 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:47:58,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:47:59,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:00,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:01,870 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:02,870 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:03,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:03,870 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:04,870 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:05,870 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:06,871 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:07,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:08,648 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:48:08,648 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f99205433ef4972a7f4b.mp4 with policy now +2025-01-18 20:48:08,649 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:48:08,651 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:48:08,651 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:48:08,652 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:48:08,872 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:08,917 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/6bg37292-media/videos/eval/1_eval_1_f99205433ef4972a7f4b.mp4 +2025-01-18 20:48:09,430 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:09,430 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:48:09,430 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/1_eval_1_f99205433ef4972a7f4b.mp4 +2025-01-18 20:48:09,430 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:48:09,661 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:09,725 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/pnystu9r-media/videos/eval/1_eval_1_f99205433ef4972a7f4b.mp4 +2025-01-18 20:48:09,873 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:10,872 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:11,430 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:11,873 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:12,466 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:48:12,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:48:12,855 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:48:12,873 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:13,431 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:13,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:14,875 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:14,884 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:15,873 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:16,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:17,432 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:17,880 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:18,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:19,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:20,380 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:20,433 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:48:20,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:21,322 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:48:21,322 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:48:21,322 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:48:21,434 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:21,434 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:48:21,875 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:22,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:23,435 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:23,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:24,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:25,435 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:25,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:25,937 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:26,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:27,436 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:27,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:48:27,855 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:48:27,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:28,875 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:29,875 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:30,875 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:31,875 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:31,914 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:32,875 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:33,876 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:34,876 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:35,876 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:36,876 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:37,149 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:37,439 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:37,876 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:38,876 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:39,876 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:40,876 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:41,877 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:42,467 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:48:42,467 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:42,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:48:42,855 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:48:42,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:43,877 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:44,877 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:45,877 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:46,877 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:47,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:47,878 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:48,878 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:49,878 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:50,878 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:51,885 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:52,233 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:48:52,234 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:48:52,234 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_f1446b1ad582539ee197.mp4 with policy now +2025-01-18 20:48:52,234 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:48:52,234 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:48:52,235 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:48:52,444 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:48:52,444 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/2_eval_2_f1446b1ad582539ee197.mp4 +2025-01-18 20:48:52,444 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:48:52,505 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/vizqbk2o-media/videos/eval/2_eval_2_f1446b1ad582539ee197.mp4 +2025-01-18 20:48:52,770 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/d31v9196-media/videos/eval/2_eval_2_f1446b1ad582539ee197.mp4 +2025-01-18 20:48:52,865 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:52,883 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:53,444 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:53,889 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:54,445 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:54,883 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:55,905 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:56,887 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:57,446 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:57,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:48:57,860 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:48:57,901 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:57,957 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:48:58,885 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:48:59,446 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:48:59,889 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:00,150 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:49:00,150 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:49:00,151 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:49:00,447 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:49:00,447 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:49:00,698 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/55qdj1rm-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:49:00,890 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:01,447 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:01,895 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:02,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:03,448 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:03,843 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:03,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:04,893 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:05,449 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:05,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:06,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:07,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:08,862 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:08,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:09,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:10,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:11,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:12,467 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:49:12,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:49:12,856 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:49:12,901 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:13,895 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:14,203 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:14,895 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:15,452 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:15,895 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:16,895 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:17,895 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:18,895 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:19,862 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:19,896 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:20,896 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:21,896 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:22,896 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:23,896 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:24,896 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:25,445 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:25,456 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:25,896 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:26,896 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:27,855 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:49:27,856 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:49:27,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:28,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:29,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:30,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:30,986 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:31,910 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:32,400 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:49:32,400 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_b0e12a1c923453fa2db6.mp4 with policy now +2025-01-18 20:49:32,404 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:49:32,405 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:49:32,405 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:49:32,405 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:49:32,458 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:49:32,458 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/3_eval_3_b0e12a1c923453fa2db6.mp4 +2025-01-18 20:49:32,459 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:49:32,661 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/d49sz6f5-media/videos/eval/3_eval_3_b0e12a1c923453fa2db6.mp4 +2025-01-18 20:49:32,901 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:32,914 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/vrmwoqgq-media/videos/eval/3_eval_3_b0e12a1c923453fa2db6.mp4 +2025-01-18 20:49:33,459 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:33,904 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:34,459 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:34,910 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:35,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:36,923 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:36,985 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:37,460 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:37,932 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:38,909 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:39,916 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:40,206 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:49:40,206 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:49:40,206 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:49:40,461 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:49:40,461 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:49:40,920 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:41,461 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:41,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:42,468 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:49:42,468 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:42,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:49:42,859 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:49:42,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:43,915 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:44,914 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:45,463 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:45,914 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:46,915 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:47,463 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:49:47,745 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:47,915 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:48,915 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:49,915 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:50,915 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:51,916 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:52,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:52,916 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:53,916 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:54,916 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:55,916 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:56,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:57,856 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:49:57,856 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:49:57,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:57,961 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:49:58,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:49:59,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:00,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:01,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:02,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:03,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:03,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:04,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:05,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:06,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:07,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:08,871 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:08,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:09,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:10,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:11,472 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:11,924 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:12,468 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:50:12,516 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:50:12,516 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_c2149e115233e8c46adb.mp4 with policy now +2025-01-18 20:50:12,520 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:50:12,520 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:50:12,520 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:50:12,520 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:50:12,799 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/gwq92w9i-media/videos/eval/4_eval_4_c2149e115233e8c46adb.mp4 +2025-01-18 20:50:12,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:50:12,860 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:50:12,924 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:13,472 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:13,473 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:50:13,473 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/4_eval_4_c2149e115233e8c46adb.mp4 +2025-01-18 20:50:13,473 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:50:13,770 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/0eeyohxv-media/videos/eval/4_eval_4_c2149e115233e8c46adb.mp4 +2025-01-18 20:50:13,930 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:13,981 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:14,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:15,938 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:16,935 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:17,474 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:17,932 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:18,930 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:19,005 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:19,944 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:20,933 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:21,475 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:21,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:22,258 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:50:22,258 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:50:22,258 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:50:22,476 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:50:22,476 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:50:22,724 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/u65nxnty-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:50:22,933 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:23,476 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:23,934 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:24,137 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:24,931 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:25,477 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:25,937 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:26,937 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:27,478 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:27,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:50:27,858 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:50:27,934 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:28,934 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:29,479 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:29,934 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:29,969 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:30,935 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:31,935 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:32,935 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:33,935 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:34,935 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:35,873 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:35,935 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:36,935 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:37,935 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:38,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:39,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:40,873 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:40,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:41,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:42,469 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:50:42,858 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:50:42,858 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:50:42,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:43,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:44,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:45,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:45,965 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:46,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:47,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:48,937 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:49,937 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:50,938 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:51,874 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:51,938 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:52,939 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:53,939 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:54,946 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:55,488 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:55,812 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:50:55,812 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_b6ffac2c1558879bc8f9.mp4 with policy now +2025-01-18 20:50:55,814 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:50:55,815 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:50:55,815 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:50:55,815 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:50:55,942 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:56,076 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/vg3pqp14-media/videos/eval/5_eval_5_b6ffac2c1558879bc8f9.mp4 +2025-01-18 20:50:56,488 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:56,488 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:50:56,488 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/5_eval_5_b6ffac2c1558879bc8f9.mp4 +2025-01-18 20:50:56,489 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:50:56,823 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/ar0jnjz5-media/videos/eval/5_eval_5_b6ffac2c1558879bc8f9.mp4 +2025-01-18 20:50:56,953 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:57,489 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:50:57,854 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:50:57,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:50:57,864 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:50:57,954 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:58,945 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:50:59,955 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:00,948 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:01,490 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:01,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:02,881 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:02,961 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:03,491 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:03,956 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:04,952 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:05,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:06,308 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:51:06,309 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:51:06,309 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:51:06,492 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:51:06,492 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:51:06,956 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:07,492 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:07,889 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:07,962 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:08,964 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:09,957 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:10,957 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:11,494 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:11,957 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:12,469 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:51:12,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:51:12,859 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:51:12,957 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:12,958 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:13,957 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:14,958 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:15,958 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:16,958 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:17,958 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:18,881 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:18,958 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:19,958 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:20,958 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:21,958 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:22,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:23,881 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:23,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:24,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:25,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:26,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:27,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:51:27,859 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:51:27,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:28,957 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:28,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:29,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:30,959 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:31,960 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:32,960 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:33,960 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:34,881 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:34,960 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:35,960 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:36,965 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:37,503 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:37,690 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:51:37,690 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_b37677ea4de36477338d.mp4 with policy now +2025-01-18 20:51:37,693 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:51:37,694 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:51:37,694 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:51:37,694 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:51:37,946 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/o099li45-media/videos/eval/6_eval_6_b37677ea4de36477338d.mp4 +2025-01-18 20:51:37,969 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:38,504 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:51:38,504 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/6_eval_6_b37677ea4de36477338d.mp4 +2025-01-18 20:51:38,504 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:51:38,811 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/v1rd2hyy-media/videos/eval/6_eval_6_b37677ea4de36477338d.mp4 +2025-01-18 20:51:38,971 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:39,504 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:39,890 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:39,973 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:40,980 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:41,981 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:42,470 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:51:42,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:51:42,869 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:51:42,974 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:43,506 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:43,983 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:44,979 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:45,257 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:45,507 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:45,993 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:46,983 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:47,508 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:51:47,746 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/q49voqrf-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:51:48,007 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:48,361 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:51:48,362 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:51:48,362 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:51:48,508 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:51:49,002 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:49,509 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:50,002 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:50,533 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:50,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:51,996 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:52,995 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:53,510 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:53,990 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:54,990 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:55,511 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:51:55,923 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:51:55,990 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:56,990 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:57,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:51:57,860 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:51:57,990 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:58,990 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:51:59,990 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:00,990 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:01,011 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:01,990 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:02,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:03,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:04,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:05,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:06,883 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:06,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:07,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:08,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:09,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:10,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:11,517 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:11,901 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:11,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:12,470 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:52:12,859 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:52:12,859 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:52:12,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:13,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:14,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:15,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:16,933 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:16,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:17,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:18,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:19,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:20,996 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:21,520 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:21,992 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:22,018 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:22,097 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:52:22,098 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:52:22,098 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_09f63e9126c49c6ebd71.mp4 with policy now +2025-01-18 20:52:22,098 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:52:22,098 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:52:22,099 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:52:22,361 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/hywi46bh-media/videos/eval/7_eval_7_09f63e9126c49c6ebd71.mp4 +2025-01-18 20:52:22,521 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:52:22,521 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/7_eval_7_09f63e9126c49c6ebd71.mp4 +2025-01-18 20:52:22,521 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:52:22,799 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/wik1nr23-media/videos/eval/7_eval_7_09f63e9126c49c6ebd71.mp4 +2025-01-18 20:52:23,012 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:23,521 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:24,003 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:24,522 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:25,007 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:26,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:27,019 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:27,523 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:27,689 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:27,868 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:52:27,868 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:52:28,027 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:29,016 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:29,524 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:30,025 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:30,413 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:52:30,414 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:52:30,461 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:52:30,525 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:52:30,525 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:52:31,025 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:31,525 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:32,026 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:32,892 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:33,034 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:33,526 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:34,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:35,021 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:35,527 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:36,029 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:37,035 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:38,022 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:38,887 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:39,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:39,528 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:52:40,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:41,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:42,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:42,470 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:52:42,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:52:42,861 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:52:43,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:43,953 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:44,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:45,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:46,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:47,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:48,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:49,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:49,887 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:50,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:51,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:52,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:53,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:54,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:54,887 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:52:55,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:56,024 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:57,025 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:57,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:52:57,861 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:52:58,025 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:59,025 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:52:59,932 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:00,025 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:01,025 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:02,025 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:03,025 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:04,026 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:05,020 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:05,049 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:05,468 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:53:05,468 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_9db24d2dbee878841c0a.mp4 with policy now +2025-01-18 20:53:05,469 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:53:05,469 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:53:05,469 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:53:05,471 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:53:05,538 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:05,538 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:53:05,538 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/8_eval_8_9db24d2dbee878841c0a.mp4 +2025-01-18 20:53:05,538 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:53:05,750 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/3np193h9-media/videos/eval/8_eval_8_9db24d2dbee878841c0a.mp4 +2025-01-18 20:53:06,028 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/pq0cnz3m-media/videos/eval/8_eval_8_9db24d2dbee878841c0a.mp4 +2025-01-18 20:53:06,031 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:07,039 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:07,539 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:08,049 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:09,031 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:10,043 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:10,173 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:11,042 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:11,540 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:12,047 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:12,471 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:53:12,867 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:53:12,867 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:53:13,055 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:13,541 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:14,059 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:15,061 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:15,973 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:16,076 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:16,469 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:53:16,470 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:53:16,470 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:53:16,542 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:53:16,542 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:53:16,787 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/gzopmazz-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:53:17,060 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:17,542 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:18,069 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:19,059 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:19,543 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:20,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:21,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:21,133 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:21,544 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:22,079 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:23,075 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:24,075 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:25,076 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:25,545 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:26,076 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:26,302 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:27,076 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:27,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:53:27,861 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:53:28,076 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:29,076 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:30,076 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:31,077 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:31,971 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:32,077 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:33,077 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:34,077 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:35,077 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:36,077 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:37,077 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:37,906 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:38,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:39,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:40,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:41,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:42,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:42,471 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:53:42,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:53:42,862 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:53:43,004 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:43,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:44,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:45,079 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:46,079 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:47,079 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:48,079 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:48,906 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:49,079 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:50,079 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:51,083 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:51,439 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:53:51,439 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_bdaa1599f7ed02f8d4c8.mp4 with policy now +2025-01-18 20:53:51,443 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:53:51,444 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:53:51,444 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:53:51,444 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:53:51,555 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:51,556 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:53:51,556 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/9_eval_9_bdaa1599f7ed02f8d4c8.mp4 +2025-01-18 20:53:51,556 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:53:51,706 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/452df9xa-media/videos/eval/9_eval_9_bdaa1599f7ed02f8d4c8.mp4 +2025-01-18 20:53:51,997 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/ce6xzab0-media/videos/eval/9_eval_9_bdaa1599f7ed02f8d4c8.mp4 +2025-01-18 20:53:52,097 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:52,556 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:53,095 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:53,556 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:53,932 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:54,102 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:55,092 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:56,099 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:57,119 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:57,558 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:53:57,884 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:53:57,884 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:53:58,100 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:59,094 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:53:59,113 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:53:59,558 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:00,120 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:01,116 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:02,111 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:03,112 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:04,118 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:04,527 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:54:04,527 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:54:04,527 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:04,527 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:54:04,560 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:54:04,560 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:54:05,117 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:05,561 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:06,121 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:07,153 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:08,122 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:09,123 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:09,562 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:10,123 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:10,286 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:11,123 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:11,563 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:12,123 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:12,472 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:54:12,862 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:54:12,863 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:54:13,123 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:14,123 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:15,123 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:15,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:16,123 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:17,123 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:17,565 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:18,124 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:19,124 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:20,124 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:20,920 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:21,124 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:22,124 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:23,124 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:24,124 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:25,124 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:25,921 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:26,124 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:27,125 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:27,862 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:54:27,863 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:54:28,125 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:29,126 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:30,126 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:31,002 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:31,126 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:32,126 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:33,126 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:34,126 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:35,127 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:35,572 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:36,156 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:36,190 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:36,501 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:54:36,501 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_e5b99072831fb49e316b.mp4 with policy now +2025-01-18 20:54:36,522 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:54:36,523 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:54:36,523 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:54:36,523 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:54:36,573 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:54:36,573 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/10_eval_10_e5b99072831fb49e316b.mp4 +2025-01-18 20:54:36,573 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:54:36,765 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/8hpk3irw-media/videos/eval/10_eval_10_e5b99072831fb49e316b.mp4 +2025-01-18 20:54:37,023 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/oaullsv2-media/videos/eval/10_eval_10_e5b99072831fb49e316b.mp4 +2025-01-18 20:54:37,142 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:37,573 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:38,150 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:38,573 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:39,145 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:40,168 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:41,154 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:41,225 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:41,575 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:42,159 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:42,472 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:54:42,897 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:54:42,897 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:54:43,172 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:44,159 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:45,159 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:45,576 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:46,184 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:46,254 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:47,179 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:48,190 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:49,195 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:50,207 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:50,579 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:54:50,585 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:54:50,585 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:54:50,585 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:54:50,830 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/nmt2nr1s-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:54:51,194 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:51,285 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:51,579 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:51,580 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:54:52,185 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:53,191 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:53,580 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:54,209 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:55,183 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:55,581 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:54:56,183 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:56,666 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:54:57,183 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:57,863 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:54:57,863 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:54:58,183 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:54:59,183 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:00,184 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:01,184 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:01,961 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:02,184 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:03,184 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:04,184 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:05,184 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:05,585 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:06,184 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:07,184 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:07,198 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:08,185 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:09,185 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:09,587 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:10,185 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:11,185 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:11,588 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:12,185 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:12,473 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:55:12,474 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:12,863 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:55:12,863 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:55:13,185 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:13,589 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:14,186 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:15,186 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:15,590 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:16,186 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:17,186 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:17,841 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:18,186 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:19,186 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:19,592 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:20,187 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:21,187 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:22,217 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:22,649 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:55:22,649 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_c3bd4b9c68e23c764392.mp4 with policy now +2025-01-18 20:55:22,657 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:55:22,657 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:55:22,658 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:55:22,658 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:55:22,901 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/ljnzpbu7-media/videos/eval/11_eval_11_c3bd4b9c68e23c764392.mp4 +2025-01-18 20:55:22,941 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:23,189 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:23,594 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:23,594 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:55:23,594 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/11_eval_11_c3bd4b9c68e23c764392.mp4 +2025-01-18 20:55:23,594 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:55:23,897 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/6a0fcxbo-media/videos/eval/11_eval_11_c3bd4b9c68e23c764392.mp4 +2025-01-18 20:55:24,222 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:25,212 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:26,228 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:27,217 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:27,595 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:27,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:55:27,894 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:55:27,974 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:28,224 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:29,234 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:30,244 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:31,248 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:31,597 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:31,597 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:55:31,635 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:55:31,635 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:55:31,641 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:55:32,240 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:32,597 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:55:33,244 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:33,597 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:33,953 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:34,233 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:35,241 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:35,598 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:36,241 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:37,241 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:37,599 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:55:38,241 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:39,242 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:39,941 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:40,241 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:41,241 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:42,242 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:42,473 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:55:42,867 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:55:42,868 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:55:43,242 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:44,242 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:44,995 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:45,242 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:46,242 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:47,243 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:48,243 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:49,243 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:50,243 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:50,941 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:51,243 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:52,243 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:53,243 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:54,243 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:55,244 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:55,942 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:55:56,244 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:57,244 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:57,867 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:55:57,868 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:55:58,244 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:55:59,244 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:00,244 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:01,191 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:01,244 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:01,609 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:02,285 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:03,261 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:56:03,261 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_941e17d058fd7b746278.mp4 with policy now +2025-01-18 20:56:03,276 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:03,284 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:56:03,285 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:56:03,285 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:56:03,285 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:56:03,523 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/ltonb2ce-media/videos/eval/12_eval_12_941e17d058fd7b746278.mp4 +2025-01-18 20:56:03,610 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:03,610 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:56:03,610 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/12_eval_12_941e17d058fd7b746278.mp4 +2025-01-18 20:56:03,611 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:56:03,873 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/wstfjh2r-media/videos/eval/12_eval_12_941e17d058fd7b746278.mp4 +2025-01-18 20:56:04,264 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:05,281 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:05,611 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:06,286 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:06,372 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:07,294 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:07,612 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:08,312 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:09,318 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:10,303 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:11,316 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:11,613 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:12,109 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:12,317 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:12,474 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:56:12,882 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:56:12,883 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:56:13,316 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:14,330 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:14,615 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:56:14,687 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:56:14,687 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:56:14,688 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:56:15,349 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:15,615 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:15,615 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:56:16,341 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:17,343 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:17,349 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:17,616 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:18,321 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:19,352 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:19,617 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:20,332 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:21,357 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:22,349 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:22,445 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:23,351 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:24,339 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:25,333 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:25,619 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:26,333 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:27,333 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:27,620 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:27,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:56:27,869 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:56:28,028 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:28,333 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:29,333 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:30,333 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:31,333 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:32,334 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:33,334 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:33,622 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:33,964 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:34,334 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:35,334 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:36,334 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:37,337 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:38,334 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:38,964 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:39,335 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:39,625 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:40,335 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:41,335 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:42,335 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:42,474 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:56:42,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:56:42,869 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:56:43,335 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:43,626 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:44,137 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:44,335 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:45,335 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:46,336 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:47,335 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:48,336 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:49,336 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:49,629 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:49,965 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:50,336 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:51,336 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:52,357 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:53,346 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:53,549 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:56:53,549 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_cf7080346708b3da4f52.mp4 with policy now +2025-01-18 20:56:53,565 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:56:53,566 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:56:53,566 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:56:53,566 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:56:53,631 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:53,631 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:56:53,631 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/13_eval_13_cf7080346708b3da4f52.mp4 +2025-01-18 20:56:53,631 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:56:53,813 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/6dfva5eu-media/videos/eval/13_eval_13_cf7080346708b3da4f52.mp4 +2025-01-18 20:56:54,080 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/aisplhnz-media/videos/eval/13_eval_13_cf7080346708b3da4f52.mp4 +2025-01-18 20:56:54,353 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:55,351 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:55,604 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:56:55,632 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:56,348 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:57,356 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:57,632 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:56:57,886 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:56:57,887 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:56:58,387 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:56:59,373 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:00,350 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:01,205 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:01,353 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:01,634 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:02,355 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:03,383 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:04,379 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:05,371 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:06,262 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:06,367 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:06,636 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:57:06,752 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:57:06,752 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:57:06,753 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:57:06,897 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/xnwnqnjg-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:57:07,374 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:07,636 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:07,637 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:57:08,376 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:09,388 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:09,637 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:10,385 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:11,392 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:11,597 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:11,638 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:12,391 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:12,475 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:57:12,894 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:57:12,894 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:57:13,397 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:14,390 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:15,401 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:16,405 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:17,388 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:17,515 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:17,640 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:18,388 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:19,388 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:20,388 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:21,389 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:22,389 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:22,984 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:23,389 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:24,390 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:25,390 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:26,390 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:27,390 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:27,877 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:57:27,877 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:57:27,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:28,390 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:29,391 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:30,391 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:31,391 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:32,391 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:33,391 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:33,984 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:34,391 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:35,392 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:36,392 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:37,392 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:38,392 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:38,984 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:39,392 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:40,392 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:41,392 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:42,393 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:42,478 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:57:42,877 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:57:42,878 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:57:43,393 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:43,652 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:44,035 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:44,418 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:44,577 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:57:44,577 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_38839f2b6468bf8a5f05.mp4 with policy now +2025-01-18 20:57:44,588 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:57:44,588 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:57:44,588 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:57:44,588 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:57:44,653 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:57:44,653 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/14_eval_14_38839f2b6468bf8a5f05.mp4 +2025-01-18 20:57:44,653 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:57:44,837 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/8niptex4-media/videos/eval/14_eval_14_38839f2b6468bf8a5f05.mp4 +2025-01-18 20:57:45,098 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/o2zm5s4w-media/videos/eval/14_eval_14_38839f2b6468bf8a5f05.mp4 +2025-01-18 20:57:45,422 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:45,653 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:46,423 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:47,441 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:47,654 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:48,462 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:49,385 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:49,441 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:49,655 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:50,451 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:51,481 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:52,465 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:53,449 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:53,656 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:57:54,470 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:55,038 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:57:55,463 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:56,470 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:57,474 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:57,904 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:57:57,904 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:57:58,477 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:57:59,506 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:00,391 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:00,485 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:00,659 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:58:00,812 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:58:00,812 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:58:00,813 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:58:01,497 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:01,660 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:58:01,660 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:58:02,495 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:03,505 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:03,661 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:58:04,514 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:05,514 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:05,661 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:58:05,777 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:06,514 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:07,530 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:08,514 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:09,522 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:10,520 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:11,033 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:11,524 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:12,475 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:58:12,516 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:12,881 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:58:12,881 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:58:13,516 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:13,665 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:58:14,516 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:15,516 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:16,516 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:17,008 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:17,516 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:18,516 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:19,517 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:20,517 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:21,517 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:22,010 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:22,517 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:23,517 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:24,517 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:25,517 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:26,517 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:27,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:27,881 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:58:27,881 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:58:27,950 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:28,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:29,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:30,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:31,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:32,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:33,009 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:33,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:34,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:35,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:36,518 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:37,519 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:38,519 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:39,009 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:39,519 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:40,535 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:41,018 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:58:41,018 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_3c41565be23b9dc07e38.mp4 with policy now +2025-01-18 20:58:41,028 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:58:41,029 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:58:41,029 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:58:41,030 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:58:41,526 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:41,677 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:58:41,677 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/15_eval_15_3c41565be23b9dc07e38.mp4 +2025-01-18 20:58:41,677 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:58:41,759 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/jbxcrffu-media/videos/eval/15_eval_15_3c41565be23b9dc07e38.mp4 +2025-01-18 20:58:42,310 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/fv1ikth9-media/videos/eval/15_eval_15_3c41565be23b9dc07e38.mp4 +2025-01-18 20:58:42,476 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:58:42,538 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:42,677 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:58:42,911 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:58:42,911 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:58:43,558 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:43,678 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:58:44,085 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:44,532 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:45,558 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:46,576 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:46,679 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:58:47,557 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:48,575 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:49,555 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:50,013 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:50,566 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:50,680 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:58:51,586 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:52,579 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:53,589 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:54,594 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:55,016 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:58:55,608 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:56,594 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:57,609 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:57,912 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:58:57,912 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:58:58,584 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:58:59,588 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:00,059 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:00,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:00,883 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:59:00,883 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:59:00,884 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:59:01,627 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:01,685 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:59:01,686 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:59:02,647 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:02,686 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:03,653 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:04,648 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:04,686 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:05,247 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:05,616 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:06,617 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:06,687 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:07,617 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:08,617 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:08,688 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:09,617 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:10,617 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:11,022 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:11,617 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:12,476 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:59:12,617 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:12,891 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:59:12,891 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:59:13,617 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:14,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:15,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:16,022 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:16,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:17,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:18,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:19,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:20,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:20,694 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:21,338 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:21,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:22,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:23,618 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:24,619 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:25,619 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:26,619 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:27,022 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:27,619 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:27,891 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:59:27,892 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:59:28,620 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:29,620 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:30,620 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:31,620 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:32,023 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:32,627 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:33,660 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:33,968 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 20:59:33,968 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_92f238d9751dde8bacfd.mp4 with policy now +2025-01-18 20:59:34,001 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:59:34,002 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:59:34,002 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:59:34,002 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:59:34,350 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/9jbv8ta0-media/videos/eval/16_eval_16_92f238d9751dde8bacfd.mp4 +2025-01-18 20:59:34,669 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:34,700 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:34,700 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:59:34,700 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/16_eval_16_92f238d9751dde8bacfd.mp4 +2025-01-18 20:59:34,700 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 20:59:35,130 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/77836vkw-media/videos/eval/16_eval_16_92f238d9751dde8bacfd.mp4 +2025-01-18 20:59:35,661 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:35,700 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:36,662 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:36,700 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:37,120 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:37,684 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:38,679 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:39,676 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:40,681 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:40,702 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:41,686 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:42,477 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 20:59:42,477 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:42,693 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:42,703 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:42,910 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:59:42,910 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:59:43,719 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:44,703 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:45,725 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:46,705 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:59:46,739 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:46,939 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 20:59:46,939 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:59:46,940 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:59:46,965 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/qjx6oikp-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:59:47,706 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 20:59:47,706 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 20:59:47,708 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:47,753 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:48,706 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:48,714 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:49,714 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:50,707 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:50,725 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:51,742 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:52,708 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:52,729 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:52,809 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:53,736 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:54,748 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:55,730 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:56,722 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:57,761 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:57,846 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:59:57,895 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:59:57,895 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:59:58,711 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 20:59:58,715 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:59:59,715 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:00,712 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:00,715 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:01,715 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:02,715 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:03,071 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:03,715 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:04,715 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:05,715 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:06,714 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:06,716 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:07,716 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:08,071 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:08,716 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:09,716 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:10,716 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:11,716 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:12,477 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 21:00:12,716 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:12,895 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:00:12,895 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:00:13,632 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:13,716 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:14,717 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:15,716 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:16,717 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:16,719 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:17,717 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:18,717 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:19,071 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:19,717 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:20,717 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:21,717 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:22,717 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:23,718 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:24,073 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:24,720 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:25,730 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:25,903 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 21:00:25,903 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_cff7eb6bb7d10755b32e.mp4 with policy now +2025-01-18 21:00:25,913 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:00:25,914 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 21:00:25,914 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:00:25,914 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:00:26,279 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/rax0mwa4-media/videos/eval/17_eval_17_cff7eb6bb7d10755b32e.mp4 +2025-01-18 21:00:26,723 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:26,723 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:00:26,723 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/17_eval_17_cff7eb6bb7d10755b32e.mp4 +2025-01-18 21:00:26,723 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 21:00:26,734 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:27,156 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/lwbly7sr-media/videos/eval/17_eval_17_cff7eb6bb7d10755b32e.mp4 +2025-01-18 21:00:27,787 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:27,936 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:00:27,936 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:00:28,724 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:28,738 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:29,108 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:29,757 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:30,761 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:31,762 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:32,725 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:32,748 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:33,793 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:34,726 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:34,756 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:35,072 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:35,745 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:36,753 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:37,768 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:38,006 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 21:00:38,006 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:00:38,007 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:00:38,728 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:00:38,729 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 21:00:38,761 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:39,759 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:40,729 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:40,772 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:40,993 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:41,766 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:42,478 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 21:00:42,794 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:42,912 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:00:42,912 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:00:43,805 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:44,731 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:44,778 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:45,800 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:46,075 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:46,794 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:47,784 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:48,783 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:49,790 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:50,803 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:51,101 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:51,789 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:52,784 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:53,784 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:54,735 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:00:54,784 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:55,784 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:56,784 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:57,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:00:57,784 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:57,902 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:00:57,902 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:00:58,784 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:00:59,785 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:00,785 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:01,785 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:02,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:02,785 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:03,785 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:04,785 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:05,785 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:06,785 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:07,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:07,786 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:08,786 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:09,786 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:10,786 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:11,786 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:12,478 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 21:01:12,478 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:12,786 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:12,902 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:01:12,902 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:01:13,786 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:14,787 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:15,787 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:16,787 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:17,787 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:18,078 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:18,787 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:19,788 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:20,390 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 21:01:20,390 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_831456511316b3120131.mp4 with policy now +2025-01-18 21:01:20,401 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:01:20,401 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 21:01:20,401 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:01:20,401 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:01:20,747 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:01:20,747 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:01:20,747 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/18_eval_18_831456511316b3120131.mp4 +2025-01-18 21:01:20,747 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 21:01:20,765 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/1622zx39-media/videos/eval/18_eval_18_831456511316b3120131.mp4 +2025-01-18 21:01:20,808 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:21,177 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/jdd06vsr-media/videos/eval/18_eval_18_831456511316b3120131.mp4 +2025-01-18 21:01:21,747 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:01:21,808 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:22,748 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:01:22,856 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:23,096 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:23,748 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:01:23,860 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:24,869 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:25,857 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:26,750 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:01:26,864 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:27,841 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:27,902 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:01:27,902 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:01:28,144 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:28,751 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:01:28,887 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:29,751 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 21:01:29,861 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:30,071 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 21:01:30,071 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:01:30,072 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:01:30,752 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:01:30,752 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 21:01:30,910 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:31,915 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:32,753 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:01:32,922 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:33,886 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:34,115 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:34,885 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:35,889 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:36,755 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:01:36,915 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:37,913 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:38,940 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:39,133 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:39,941 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:40,913 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:41,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:42,479 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 21:01:42,927 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:42,946 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:01:42,946 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:01:43,924 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:44,922 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:44,979 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:45,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:46,760 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:01:46,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:47,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:48,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:49,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:50,114 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:50,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:51,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:52,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:53,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:54,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:55,917 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:56,114 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:01:56,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:57,902 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:01:57,903 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:01:57,945 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:58,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:01:59,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:00,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:01,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:02,114 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:02,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:03,918 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:04,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:05,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:06,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:07,114 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:07,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:08,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:09,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:10,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:11,919 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:12,152 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:12,479 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 21:02:12,948 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:12,949 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:02:12,949 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:02:13,935 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 21:02:13,935 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_cc714c15f81700a6ca67.mp4 with policy now +2025-01-18 21:02:13,939 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:13,963 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:02:13,963 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 21:02:13,963 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:13,964 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:14,334 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/pq6hgnnl-media/videos/eval/19_eval_19_cc714c15f81700a6ca67.mp4 +2025-01-18 21:02:14,773 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:02:14,773 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:02:14,773 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/19_eval_19_cc714c15f81700a6ca67.mp4 +2025-01-18 21:02:14,773 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval +2025-01-18 21:02:14,951 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:15,188 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/iwmah4tp-media/videos/eval/19_eval_19_cc714c15f81700a6ca67.mp4 +2025-01-18 21:02:15,773 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:02:15,942 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:16,774 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:02:16,991 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:17,675 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:17,993 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:18,983 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:19,997 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:20,776 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:02:20,988 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:21,986 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:22,777 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:02:23,019 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:23,146 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:24,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:25,069 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:26,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:26,778 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 21:02:27,022 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:27,140 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 21:02:27,141 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:27,141 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:27,779 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:02:27,949 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:02:27,949 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:02:28,040 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:28,121 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 21:02:28,121 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 21:02:28,351 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:02:28,626 INFO wandb-upload_1:2573666 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4oyofpyi +2025-01-18 21:02:28,713 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 21:02:28,722 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 21:02:28,779 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:02:28,780 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 21:02:28,780 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/table/eval/total_success_rate_ci_21_7c95301d486f1fc093c2.table.json +2025-01-18 21:02:28,780 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/table/eval/success_rate_ci_20_626687eda62aee1b0486.table.json +2025-01-18 21:02:28,780 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media +2025-01-18 21:02:28,780 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/table/eval +2025-01-18 21:02:28,780 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/table +2025-01-18 21:02:29,013 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:29,028 INFO SenderThread:2573666 [sender.py:send_request_log_artifact():1525] logged artifact run-7sdjk1lj-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MjAyMDY3MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzEwMDk1Mg==', 'latestArtifact': None}} +2025-01-18 21:02:29,028 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 21:02:29,029 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_626687eda62aee1b0486.table.json with policy now +2025-01-18 21:02:29,029 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:29,029 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 21:02:29,029 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,030 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,031 DEBUG SenderThread:2573666 [sender.py:send():379] send: telemetry +2025-01-18 21:02:29,032 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,034 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,034 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,035 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,035 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,035 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,036 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,036 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,036 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,037 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,037 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,037 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,037 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,038 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,038 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,039 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,039 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,039 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,040 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,040 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,040 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,041 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:29,041 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 21:02:29,277 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/7n3d5rvk-media/table/eval/success_rate_ci_20_626687eda62aee1b0486.table.json +2025-01-18 21:02:29,279 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/o4blrnq1-media/table/eval/total_success_rate_ci_21_7c95301d486f1fc093c2.table.json +2025-01-18 21:02:29,301 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/84uo7ur2-events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 21:02:29,419 INFO wandb-upload_1:2573666 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpu2xsesb8 +2025-01-18 21:02:29,572 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/zewd7psa-media/table/eval/success_rate_ci_20_626687eda62aee1b0486.table.json +2025-01-18 21:02:29,780 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:02:29,838 INFO SenderThread:2573666 [sender.py:send_request_log_artifact():1525] logged artifact run-7sdjk1lj-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MjAyMDY3Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzEwMDk1Mw==', 'latestArtifact': None}} +2025-01-18 21:02:29,838 DEBUG SenderThread:2573666 [sender.py:send():379] send: files +2025-01-18 21:02:29,838 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_7c95301d486f1fc093c2.table.json with policy now +2025-01-18 21:02:29,838 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 21:02:29,838 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:02:29,839 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:02:30,013 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:30,096 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /tmp/tmpr8tym8tcwandb/mfwizwuz-media/table/eval/total_success_rate_ci_21_7c95301d486f1fc093c2.table.json +2025-01-18 21:02:30,781 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:02:30,781 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:02:31,013 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:32,013 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:33,013 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:34,013 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:34,135 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:35,014 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:36,014 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:37,014 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:38,014 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:39,014 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:39,135 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:40,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:41,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:42,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:42,479 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 21:02:42,915 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:02:42,916 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:02:43,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:44,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:44,135 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:45,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:46,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:47,015 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:48,016 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:49,016 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:50,016 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:50,135 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:51,016 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:52,016 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:53,016 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:54,017 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:55,017 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:55,138 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:02:55,793 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/config.yaml +2025-01-18 21:02:56,017 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:57,017 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:57,916 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 21:02:57,916 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: stop_status +2025-01-18 21:02:58,017 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:02:59,017 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:00,017 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:01,018 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:01,136 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:03:02,018 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:03,018 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:03,582 DEBUG SenderThread:2573666 [sender.py:send():379] send: exit +2025-01-18 21:03:03,582 INFO SenderThread:2573666 [sender.py:send_exit():586] handling exit code: 0 +2025-01-18 21:03:03,582 INFO SenderThread:2573666 [sender.py:send_exit():588] handling runtime: 1011 +2025-01-18 21:03:03,582 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:03:03,582 INFO SenderThread:2573666 [sender.py:send_exit():594] send defer +2025-01-18 21:03:03,582 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:03,582 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-18 21:03:03,583 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:03,583 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-18 21:03:03,583 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 1 +2025-01-18 21:03:03,583 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:03,583 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-18 21:03:03,583 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:03,583 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-18 21:03:03,583 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 2 +2025-01-18 21:03:03,583 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:03,583 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-18 21:03:03,583 INFO HandlerThread:2573666 [system_monitor.py:finish():203] Stopping system monitor +2025-01-18 21:03:03,584 DEBUG SystemMonitor:2573666 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-18 21:03:03,584 DEBUG SystemMonitor:2573666 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-18 21:03:03,584 INFO HandlerThread:2573666 [interfaces.py:finish():200] Joined cpu monitor +2025-01-18 21:03:03,585 INFO HandlerThread:2573666 [interfaces.py:finish():200] Joined disk monitor +2025-01-18 21:03:03,637 INFO HandlerThread:2573666 [interfaces.py:finish():200] Joined gpu monitor +2025-01-18 21:03:03,637 INFO HandlerThread:2573666 [interfaces.py:finish():200] Joined memory monitor +2025-01-18 21:03:03,637 INFO HandlerThread:2573666 [interfaces.py:finish():200] Joined network monitor +2025-01-18 21:03:03,638 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:03,638 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-18 21:03:03,638 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 3 +2025-01-18 21:03:03,638 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:03,638 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-18 21:03:03,643 DEBUG SenderThread:2573666 [sender.py:send():379] send: stats +2025-01-18 21:03:03,643 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:03,643 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-18 21:03:03,643 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 4 +2025-01-18 21:03:03,643 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:03,643 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-18 21:03:03,797 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:03:04,798 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:03:10,151 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:03:10,152 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:03:10,152 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: keepalive +2025-01-18 21:03:10,153 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:10,153 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-18 21:03:10,153 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 5 +2025-01-18 21:03:10,153 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:03:10,153 DEBUG SenderThread:2573666 [sender.py:send():379] send: history +2025-01-18 21:03:10,153 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: summary_record +2025-01-18 21:03:10,153 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:10,153 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-18 21:03:10,154 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:03:10,154 DEBUG SenderThread:2573666 [sender.py:send():379] send: summary +2025-01-18 21:03:10,158 INFO SenderThread:2573666 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 21:03:10,158 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:10,158 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-18 21:03:10,158 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 6 +2025-01-18 21:03:10,159 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:10,159 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-18 21:03:10,159 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:10,159 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-18 21:03:10,159 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 7 +2025-01-18 21:03:10,159 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 21:03:10,159 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:10,159 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-18 21:03:10,159 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:10,159 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-18 21:03:10,583 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:03:10,801 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:03:13,018 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 8 +2025-01-18 21:03:13,018 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:03:13,018 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:13,018 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-18 21:03:13,019 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:13,019 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-18 21:03:13,019 INFO SenderThread:2573666 [job_builder.py:build():440] Attempting to build job artifact +2025-01-18 21:03:13,022 INFO SenderThread:2573666 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-18 21:03:13,024 INFO SenderThread:2573666 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-18 21:03:13,039 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 9 +2025-01-18 21:03:13,039 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:13,039 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-18 21:03:13,039 DEBUG SenderThread:2573666 [sender.py:send():379] send: artifact +2025-01-18 21:03:13,404 INFO wandb-upload_2:2573666 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpnfp61zup +2025-01-18 21:03:13,406 INFO wandb-upload_3:2573666 [upload_job.py:push():85] Skipped uploading /tmp/tmp3mer6rfj/wandb-job.json +2025-01-18 21:03:13,519 INFO wandb-upload_1:2573666 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprfodh2rt +2025-01-18 21:03:13,584 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:03:13,803 INFO Thread-12 :2573666 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:03:14,226 INFO SenderThread:2573666 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1MjAyMTE2MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1MTk1NDMyOQ==', 'versionIndex': 449}}} +2025-01-18 21:03:14,226 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:14,227 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-18 21:03:14,227 INFO SenderThread:2573666 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-18 21:03:14,804 INFO SenderThread:2573666 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files +2025-01-18 21:03:14,804 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/config.yaml config.yaml +2025-01-18 21:03:14,804 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 events.out.tfevents.1737251172.leonmkim-ROG-Strix-G15CS-G15CS.2573483.0 +2025-01-18 21:03:14,804 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json wandb-summary.json +2025-01-18 21:03:14,804 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/requirements.txt requirements.txt +2025-01-18 21:03:14,804 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-metadata.json wandb-metadata.json +2025-01-18 21:03:14,804 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 21:03:14,804 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log output.log +2025-01-18 21:03:14,806 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/diff.patch diff.patch +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/2_eval_2_f1446b1ad582539ee197.mp4 media/videos/eval/2_eval_2_f1446b1ad582539ee197.mp4 +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/14_eval_14_38839f2b6468bf8a5f05.mp4 media/videos/eval/14_eval_14_38839f2b6468bf8a5f05.mp4 +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/7_eval_7_09f63e9126c49c6ebd71.mp4 media/videos/eval/7_eval_7_09f63e9126c49c6ebd71.mp4 +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/10_eval_10_e5b99072831fb49e316b.mp4 media/videos/eval/10_eval_10_e5b99072831fb49e316b.mp4 +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/4_eval_4_c2149e115233e8c46adb.mp4 media/videos/eval/4_eval_4_c2149e115233e8c46adb.mp4 +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/5_eval_5_b6ffac2c1558879bc8f9.mp4 media/videos/eval/5_eval_5_b6ffac2c1558879bc8f9.mp4 +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/3_eval_3_b0e12a1c923453fa2db6.mp4 media/videos/eval/3_eval_3_b0e12a1c923453fa2db6.mp4 +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/17_eval_17_cff7eb6bb7d10755b32e.mp4 media/videos/eval/17_eval_17_cff7eb6bb7d10755b32e.mp4 +2025-01-18 21:03:14,811 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/19_eval_19_cc714c15f81700a6ca67.mp4 media/videos/eval/19_eval_19_cc714c15f81700a6ca67.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/9_eval_9_bdaa1599f7ed02f8d4c8.mp4 media/videos/eval/9_eval_9_bdaa1599f7ed02f8d4c8.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/8_eval_8_9db24d2dbee878841c0a.mp4 media/videos/eval/8_eval_8_9db24d2dbee878841c0a.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/16_eval_16_92f238d9751dde8bacfd.mp4 media/videos/eval/16_eval_16_92f238d9751dde8bacfd.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/12_eval_12_941e17d058fd7b746278.mp4 media/videos/eval/12_eval_12_941e17d058fd7b746278.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/1_eval_1_f99205433ef4972a7f4b.mp4 media/videos/eval/1_eval_1_f99205433ef4972a7f4b.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/11_eval_11_c3bd4b9c68e23c764392.mp4 media/videos/eval/11_eval_11_c3bd4b9c68e23c764392.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/15_eval_15_3c41565be23b9dc07e38.mp4 media/videos/eval/15_eval_15_3c41565be23b9dc07e38.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/13_eval_13_cf7080346708b3da4f52.mp4 media/videos/eval/13_eval_13_cf7080346708b3da4f52.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/0_eval_0_8f82b238c966cdc59610.mp4 media/videos/eval/0_eval_0_8f82b238c966cdc59610.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/18_eval_18_831456511316b3120131.mp4 media/videos/eval/18_eval_18_831456511316b3120131.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/videos/eval/6_eval_6_b37677ea4de36477338d.mp4 media/videos/eval/6_eval_6_b37677ea4de36477338d.mp4 +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/table/eval/success_rate_ci_20_626687eda62aee1b0486.table.json media/table/eval/success_rate_ci_20_626687eda62aee1b0486.table.json +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/media/table/eval/total_success_rate_ci_21_7c95301d486f1fc093c2.table.json media/table/eval/total_success_rate_ci_21_7c95301d486f1fc093c2.table.json +2025-01-18 21:03:14,812 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 10 +2025-01-18 21:03:14,813 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:03:14,813 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:14,813 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-18 21:03:14,813 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:14,813 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-18 21:03:14,813 INFO SenderThread:2573666 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 21:03:14,986 INFO wandb-upload_3:2573666 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/requirements.txt +2025-01-18 21:03:14,993 INFO wandb-upload_2:2573666 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/wandb-summary.json +2025-01-18 21:03:15,002 INFO wandb-upload_0:2573666 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/config.yaml +2025-01-18 21:03:15,021 INFO wandb-upload_1:2573666 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/files/output.log +2025-01-18 21:03:15,221 INFO Thread-11 (_thread_body):2573666 [sender.py:transition_state():614] send defer: 11 +2025-01-18 21:03:15,222 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:15,222 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-18 21:03:15,222 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:15,222 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-18 21:03:15,222 INFO SenderThread:2573666 [file_pusher.py:join():175] waiting for file pusher +2025-01-18 21:03:15,225 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 12 +2025-01-18 21:03:15,225 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:15,225 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-18 21:03:15,225 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:15,225 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-18 21:03:15,225 INFO SenderThread:2573666 [file_stream.py:finish():601] file stream finish called +2025-01-18 21:03:15,297 INFO SenderThread:2573666 [file_stream.py:finish():605] file stream finish is done +2025-01-18 21:03:15,297 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 13 +2025-01-18 21:03:15,297 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:15,297 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-18 21:03:15,298 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:15,298 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-18 21:03:15,298 INFO SenderThread:2573666 [sender.py:transition_state():614] send defer: 14 +2025-01-18 21:03:15,298 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: defer +2025-01-18 21:03:15,298 INFO HandlerThread:2573666 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-18 21:03:15,298 DEBUG SenderThread:2573666 [sender.py:send():379] send: final +2025-01-18 21:03:15,298 DEBUG SenderThread:2573666 [sender.py:send():379] send: footer +2025-01-18 21:03:15,298 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: defer +2025-01-18 21:03:15,298 INFO SenderThread:2573666 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-18 21:03:15,299 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:03:15,299 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: server_info +2025-01-18 21:03:15,299 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:03:15,299 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 21:03:15,299 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: server_info +2025-01-18 21:03:15,300 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: get_summary +2025-01-18 21:03:15,301 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-18 21:03:15,302 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 21:03:15,367 DEBUG SenderThread:2573666 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 21:03:15,368 INFO MainThread:2573666 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-18 21:03:15,368 INFO MainThread:2573666 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-18 21:03:15,369 INFO MainThread:2573666 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-18 21:03:15,369 DEBUG HandlerThread:2573666 [handler.py:handle_request():158] handle_request: shutdown +2025-01-18 21:03:15,369 INFO HandlerThread:2573666 [handler.py:finish():882] shutting down handler +2025-01-18 21:03:16,300 INFO WriterThread:2573666 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/run-7sdjk1lj.wandb +2025-01-18 21:03:16,368 INFO SenderThread:2573666 [sender.py:finish():1615] shutting down sender +2025-01-18 21:03:16,368 INFO SenderThread:2573666 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 21:03:16,368 INFO SenderThread:2573666 [file_pusher.py:join():175] waiting for file pusher diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/logs/debug.log b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f3e34b33e2e0768549add31a05ca468316aaa1fb --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_setup.py:_flush():76] Configure stats pid to 2573483 +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/settings +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/logs/debug.log +2025-01-18 20:46:12,095 INFO MainThread:2573483 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/logs/debug-internal.log +2025-01-18 20:46:12,096 INFO MainThread:2573483 [wandb_init.py:init():569] calling init triggers +2025-01-18 20:46:12,096 INFO MainThread:2573483 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_1', 'wandb_notes': '3465_1_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj'} +2025-01-18 20:46:12,096 INFO MainThread:2573483 [wandb_init.py:init():619] starting backend +2025-01-18 20:46:12,096 INFO MainThread:2573483 [wandb_init.py:init():623] setting up manager +2025-01-18 20:46:12,102 INFO MainThread:2573483 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-18 20:46:12,104 INFO MainThread:2573483 [wandb_init.py:init():631] backend started and connected +2025-01-18 20:46:12,115 INFO MainThread:2573483 [wandb_init.py:init():720] updated telemetry +2025-01-18 20:46:12,121 INFO MainThread:2573483 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-18 20:46:12,412 INFO MainThread:2573483 [wandb_run.py:_on_init():2435] communicating current version +2025-01-18 20:46:12,452 INFO MainThread:2573483 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-18 20:46:12,452 INFO MainThread:2573483 [wandb_init.py:init():804] starting run threads in backend +2025-01-18 20:46:12,854 INFO MainThread:2573483 [wandb_run.py:_console_start():2413] atexit reg +2025-01-18 20:46:12,854 INFO MainThread:2573483 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-18 20:46:12,854 INFO MainThread:2573483 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-18 20:46:12,854 INFO MainThread:2573483 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-18 20:46:12,855 INFO MainThread:2573483 [wandb_init.py:init():847] run started, returning control to user process +2025-01-18 20:46:12,856 INFO MainThread:2573483 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/7sdjk1lj/tb, True +2025-01-18 20:46:30,268 INFO MainThread:2573483 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-18 21:03:16,369 WARNING MsgRouterThr:2573483 [router.py:message_loop():77] message_loop has been closed diff --git a/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/run-7sdjk1lj.wandb b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/run-7sdjk1lj.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5126d2b37746b1e1225f47d022a27e72fb2f8c2e --- /dev/null +++ b/7sdjk1lj/wandb/run-20250118_204612-7sdjk1lj/run-7sdjk1lj.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fed1813feb959f61802601246dd827a7bedb6f282eed2d9787c07818c09d646b +size 238104 diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3bb651e39ddd52fceadc722d068c63c5685b6ba --- /dev/null +++ b/config.yaml @@ -0,0 +1,573 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 1 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '3465_1' +wandb_notes: 3465_1_restarted_1 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: false +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 7 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +restore_missing_checkpoint: null +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ${test.root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${test.name_of_expert_demo} + expert_dataset: ${test.expert_dataset_dirpath}/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ${test.expert_dataset_dirpath}/${test.semantic_demo_grouping_name} + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: ${test.local_snapshot_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name} + resume_wandb_run: false +feature_type: 180x240_1_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..759b5e92e5d46706b61341cbc016177b9a54f60c --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7eab7003399cc2b1ea7cb78e460c88d14d92a244d9bb1865e95602eb62247ad +size 909989846 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..add01aa691b8fe264797f8b43bcebfde6c4e2dc8 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac5ec8ad67b551a1a5aa7c2fdb368fcd75b725200fa8b991d20ceec7b62d6e3a +size 909989846 diff --git a/snapshot_12499.pt b/snapshot_12499.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c36b59e88d59a0d245e070baff7b8f8bc46c051 --- /dev/null +++ b/snapshot_12499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61235303f84c1d6495b53354e8894541d8de2ca383a0c028b23fc066575180aa +size 909989846 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..b0a68c5703d78f7d1ec6f5a5a606345f38858fc6 --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb2f893063900702ab9048180961f244323ee883c4e120147b4cb812bc64a659 +size 909989846 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..08e0c1257d5c2b64b38f49d25305a0a090cdd4bc --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d61f668971083dad6e641b6f916e0438c4b433a29203150ddfd2586fe40b4873 +size 909989846 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..1333b2d9e0efb47fe0955e34218f19f912e37ff5 --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c13b4805c9a467165647c36e7dfc321267635e04cd99f63926efe96204ecdc0 +size 909989336 diff --git a/snapshot_5249.pt b/snapshot_5249.pt new file mode 100644 index 0000000000000000000000000000000000000000..2a2bceee7eefec87febdbfcab80487f12e714e09 --- /dev/null +++ b/snapshot_5249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6e7c2320306d04c7028d1cba39aefc2d206b1f004c166c441ed4bbd811eee67 +size 909989336 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..47ae556dfd5357fc783d90f13adcc91fc92e39b4 --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e930d1c1ba93f6d7f6ff86dfbba1ba05a47e5cb35905e4b87814d059ff66d9c +size 909989336 diff --git a/snapshot_6499.pt b/snapshot_6499.pt new file mode 100644 index 0000000000000000000000000000000000000000..6ab044f068d09e12ab0874c2140e999aefea6ee2 --- /dev/null +++ b/snapshot_6499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02b171e7cac7e4bd5b00fec6cebe50b3e73ebcfcad711b64be3c6a38c5488140 +size 909989336 diff --git a/snapshot_7249.pt b/snapshot_7249.pt new file mode 100644 index 0000000000000000000000000000000000000000..e0df63a5e92cfa348a92f48930655e0580aba5b3 --- /dev/null +++ b/snapshot_7249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c99c4f6f71b155ef8b2a6832ca35e7707075f3dea548638b938f23035893bf1 +size 909989336 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..d2d9d458186e010995eeecf949cb3478cdf3a2ee --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de32b4856f183c53912843fdebee11705caecd22fef10047aa699972b1fb9a18 +size 909989336 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f74f578585417a0ede8faf4259d03459412db13 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4a16bc396295370b31e63c04cfecf9994ca6a50fa611e710b6aee9df134e9a3 +size 909989846 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..08e0c1257d5c2b64b38f49d25305a0a090cdd4bc --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d61f668971083dad6e641b6f916e0438c4b433a29203150ddfd2586fe40b4873 +size 909989846 diff --git a/u3lo36eq/.hydra/config.yaml b/u3lo36eq/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e01202c5bb687b2213c48829aa3d8f422bf0d242 --- /dev/null +++ b/u3lo36eq/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '3465_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/u3lo36eq/.hydra/hydra.yaml b/u3lo36eq/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6748b4788af3f2d51dacc7d1c810de0962670ce0 --- /dev/null +++ b/u3lo36eq/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/u3lo36eq/.hydra/overrides.yaml b/u3lo36eq/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/u3lo36eq/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/u3lo36eq/episode_rosbags/aligned_depth_to_color_K.npy b/u3lo36eq/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/u3lo36eq/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/u3lo36eq/episode_rosbags/cam_tf_world.npy b/u3lo36eq/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/u3lo36eq/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/u3lo36eq/episode_rosbags/color_K.npy b/u3lo36eq/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/u3lo36eq/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/u3lo36eq/episode_rosbags/depth_K.npy b/u3lo36eq/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/u3lo36eq/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/u3lo36eq/episode_rosbags/episode_0_2025-01-18-19-09-22.bag b/u3lo36eq/episode_rosbags/episode_0_2025-01-18-19-09-22.bag new file mode 100644 index 0000000000000000000000000000000000000000..618a6ae3e29015c64558158e635a9593550d4c43 --- /dev/null +++ b/u3lo36eq/episode_rosbags/episode_0_2025-01-18-19-09-22.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a91f42775e4618a8183532ae1c462b9ad6507c33010663d76e5fee563b410d0a +size 1286303389 diff --git a/u3lo36eq/episode_rosbags/episode_10_2025-01-18-19-17-15.bag b/u3lo36eq/episode_rosbags/episode_10_2025-01-18-19-17-15.bag new file mode 100644 index 0000000000000000000000000000000000000000..10ee2daceba9799a69d6928c8a23f2664af2524d --- /dev/null +++ b/u3lo36eq/episode_rosbags/episode_10_2025-01-18-19-17-15.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1db93e7a64c0d85444249bc8c075a87f970ca063bbdc6049dd1cc4a921f01e47 +size 1412884249 diff --git a/u3lo36eq/episode_rosbags/episode_11_2025-01-18-19-18-17.bag b/u3lo36eq/episode_rosbags/episode_11_2025-01-18-19-18-17.bag new file mode 100644 index 0000000000000000000000000000000000000000..03838f5d62bde876a5ee6ba29a710029a35b3526 --- /dev/null +++ b/u3lo36eq/episode_rosbags/episode_11_2025-01-18-19-18-17.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4171d7a090f9661a1838067823c56dbfe8325d8a09bfbfdf165f9fe0782a9269 +size 1181785703 diff --git a/u3lo36eq/episode_rosbags/episode_12_2025-01-18-19-19-01.bag b/u3lo36eq/episode_rosbags/episode_12_2025-01-18-19-19-01.bag new file mode 100644 index 0000000000000000000000000000000000000000..1b0f87644f6753f56a7caa1c62c6515a40f25b8e --- /dev/null +++ b/u3lo36eq/episode_rosbags/episode_12_2025-01-18-19-19-01.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bf5e00bc388c7d74491711f71ba871718949bb8f5f2d77b4af314b6dfeee418 +size 1413020473 diff --git a/u3lo36eq/episode_rosbags/episode_13_2025-01-18-19-19-43.bag b/u3lo36eq/episode_rosbags/episode_13_2025-01-18-19-19-43.bag new file mode 100644 index 0000000000000000000000000000000000000000..a0993937ca43711113cb1b4ca312654de741e162 --- /dev/null +++ b/u3lo36eq/episode_rosbags/episode_13_2025-01-18-19-19-43.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c0d93cedc0a2af42ecee9be73cc81a31d336b39e2ed7b758c102fbd724938fa +size 1413056881 diff --git a/u3lo36eq/episode_rosbags/episode_14_2025-01-18-19-21-30.bag b/u3lo36eq/episode_rosbags/episode_14_2025-01-18-19-21-30.bag new file mode 100644 index 0000000000000000000000000000000000000000..09365743170c72c4bcca04caf36595f42972b603 --- /dev/null +++ b/u3lo36eq/episode_rosbags/episode_14_2025-01-18-19-21-30.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0055f0f764744ab947fb574e60370aa3e92533d85135874f394ab534d71a08e +size 1411318976 diff --git a/u3lo36eq/episode_rosbags/episode_16_2025-01-18-19-23-20.bag b/u3lo36eq/episode_rosbags/episode_16_2025-01-18-19-23-20.bag new file mode 100644 index 0000000000000000000000000000000000000000..88b2bafe8caff08283487832f37175f9ae886246 --- /dev/null +++ b/u3lo36eq/episode_rosbags/episode_16_2025-01-18-19-23-20.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed21b0ea5e5b9e6adce7b60888d34d5a644e2769e643b11c3145091708ccaa45 +size 1411841084 diff --git a/u3lo36eq/episode_rosbags/episode_17_2025-01-18-19-24-16.bag b/u3lo36eq/episode_rosbags/episode_17_2025-01-18-19-24-16.bag new file mode 100644 index 0000000000000000000000000000000000000000..7542fc6740fc128def98e91f4ce1b252711ca166 --- /dev/null +++ b/u3lo36eq/episode_rosbags/episode_17_2025-01-18-19-24-16.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0444b4486e70b2ab38fefe1183a7ca927ae9c2aa3c79b0943f999abf83cc36b +size 1407884721 diff --git a/u3lo36eq/episode_rosbags/episode_18_2025-01-18-19-25-14.bag b/u3lo36eq/episode_rosbags/episode_18_2025-01-18-19-25-14.bag new file mode 100644 index 0000000000000000000000000000000000000000..35f88a3eff7854d2b266588c4d9c05f400588aa0 --- /dev/null +++ b/u3lo36eq/episode_rosbags/episode_18_2025-01-18-19-25-14.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f19eb202f3d96656cab3fc772e0e8ab553440f859e341df14270608af11a0466 +size 1233607834 diff --git a/u3lo36eq/episode_rosbags/episode_19_2025-01-18-19-26-02.bag b/u3lo36eq/episode_rosbags/episode_19_2025-01-18-19-26-02.bag new file mode 100644 index 0000000000000000000000000000000000000000..b40e9477f5f0b76afb666aad18205bc3cec83d34 --- /dev/null +++ b/u3lo36eq/episode_rosbags/episode_19_2025-01-18-19-26-02.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee9d4d62326170bc623e3500e178d1f1cd7d906778937e134268984c50ad697a +size 1412097369 diff --git a/u3lo36eq/episode_rosbags/episode_1_2025-01-18-19-10-10.bag b/u3lo36eq/episode_rosbags/episode_1_2025-01-18-19-10-10.bag new file mode 100644 index 0000000000000000000000000000000000000000..65e6dd4fe47e637d4737935e43035220d3cf1a92 --- /dev/null +++ b/u3lo36eq/episode_rosbags/episode_1_2025-01-18-19-10-10.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bff80659a9f504c62bfa98cb2277b925cc77ebcb3fc8bf259c599553f8ca82b7 +size 1229508562 diff --git a/u3lo36eq/episode_rosbags/episode_2_2025-01-18-19-10-48.bag b/u3lo36eq/episode_rosbags/episode_2_2025-01-18-19-10-48.bag new file mode 100644 index 0000000000000000000000000000000000000000..7029261912b7dd1eace7c95d99ded9012ab58f47 --- /dev/null +++ b/u3lo36eq/episode_rosbags/episode_2_2025-01-18-19-10-48.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe3ea5f6e6965a42ebf8ea279a451239888828f921df80e5af32868bfb5531e4 +size 1266318608 diff --git a/u3lo36eq/episode_rosbags/episode_3_2025-01-18-19-11-30.bag b/u3lo36eq/episode_rosbags/episode_3_2025-01-18-19-11-30.bag new file mode 100644 index 0000000000000000000000000000000000000000..b57b710c603b985f140e13da676880bd0dd45e44 --- /dev/null +++ b/u3lo36eq/episode_rosbags/episode_3_2025-01-18-19-11-30.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96ee9d1c8f8bb680f5601180b11fde3659d0a109d03b24574bfb8c87d2783b10 +size 1412630996 diff --git a/u3lo36eq/episode_rosbags/episode_4_2025-01-18-19-12-21.bag b/u3lo36eq/episode_rosbags/episode_4_2025-01-18-19-12-21.bag new file mode 100644 index 0000000000000000000000000000000000000000..8668d7ec51858ac9688af7258d939bf7f89f0509 --- /dev/null +++ b/u3lo36eq/episode_rosbags/episode_4_2025-01-18-19-12-21.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f064f7a101e8ac646fc98d70343f50c47a0b607b97a8b5cd9fbb1afc2b1caab +size 1412062600 diff --git a/u3lo36eq/episode_rosbags/episode_5_2025-01-18-19-13-04.bag b/u3lo36eq/episode_rosbags/episode_5_2025-01-18-19-13-04.bag new file mode 100644 index 0000000000000000000000000000000000000000..e95eb66fc9a69439c5cb240f69f3005cb3c41983 --- /dev/null +++ b/u3lo36eq/episode_rosbags/episode_5_2025-01-18-19-13-04.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4eb0f29aad5fca25285bd206a78993e82da9ab1cc41dc08df54f5ae318e0318 +size 1413969474 diff --git a/u3lo36eq/episode_rosbags/episode_6_2025-01-18-19-13-48.bag b/u3lo36eq/episode_rosbags/episode_6_2025-01-18-19-13-48.bag new file mode 100644 index 0000000000000000000000000000000000000000..a6046380eb38c4b41690438fff2cd936e7467e33 --- /dev/null +++ b/u3lo36eq/episode_rosbags/episode_6_2025-01-18-19-13-48.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3453f0812b80ed3bc614800a87a820c8e2c630d7ddb34d77d4a76ea2bcfa4674 +size 1412018372 diff --git a/u3lo36eq/episode_rosbags/episode_8_2025-01-18-19-15-26.bag b/u3lo36eq/episode_rosbags/episode_8_2025-01-18-19-15-26.bag new file mode 100644 index 0000000000000000000000000000000000000000..4cdab425c82250c88b7ca4b569530e05c85dad05 --- /dev/null +++ b/u3lo36eq/episode_rosbags/episode_8_2025-01-18-19-15-26.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d013462c927874118a4dc63464993ff9d9a27fa495c917900ae9af6a076bf9e4 +size 1410687784 diff --git a/u3lo36eq/eval_robot.log b/u3lo36eq/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..cf255e379c7acaa959178e03afcf1fb6e12e3840 --- /dev/null +++ b/u3lo36eq/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-18 19:08:42,876][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-18 19:08:42,879][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/u3lo36eq/eval_video/0_eval.mp4 b/u3lo36eq/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a9fe5ec4e5a2da6c00833eda455c5e27e5e6d8cd --- /dev/null +++ b/u3lo36eq/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2d24d1f27d7648055dc3ada89105affb484179f0f09c9ba6f5d21d98c644d86 +size 1105784 diff --git a/u3lo36eq/eval_video/10_eval.mp4 b/u3lo36eq/eval_video/10_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..000b91b6798cfa604bda69384d68506c33599401 --- /dev/null +++ b/u3lo36eq/eval_video/10_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:741429cbea5a900591e5d28a98fc2f0646f8799905bfaac36afaa369de494fb9 +size 1257587 diff --git a/u3lo36eq/eval_video/11_eval.mp4 b/u3lo36eq/eval_video/11_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0898f77539ba27a49c4331fd701682f755eeaf51 --- /dev/null +++ b/u3lo36eq/eval_video/11_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d5f6eccd1ac8a9641c2ffa7c1563acbf5f91d531cbaf256c2447fb51ca55bb2 +size 1001294 diff --git a/u3lo36eq/eval_video/12_eval.mp4 b/u3lo36eq/eval_video/12_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2446088d54a2872905494b95a05033a7b42850f2 --- /dev/null +++ b/u3lo36eq/eval_video/12_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c363b6170975b374255612516dd2d909c80411d50b06f4ac9b21d84b2f795ec +size 1175492 diff --git a/u3lo36eq/eval_video/13_eval.mp4 b/u3lo36eq/eval_video/13_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..51c04ec9faa21b2246e2119cac4f062cee203c0f --- /dev/null +++ b/u3lo36eq/eval_video/13_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c552251a3cda7642b7546b8d1335e9ed0a120afbba48e3e84488db805833ef1c +size 1183827 diff --git a/u3lo36eq/eval_video/14_eval.mp4 b/u3lo36eq/eval_video/14_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e1bfe999aa15902b6f04ce3750c2d260f5c2c736 --- /dev/null +++ b/u3lo36eq/eval_video/14_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58248ea2be38c14379cda368756a3bb8731c389eeea3f5b49e12c09c7d7644bd +size 1455691 diff --git a/u3lo36eq/eval_video/15_eval.mp4 b/u3lo36eq/eval_video/15_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5c1173451b6072d3cc96280771b6f9af44c1026f --- /dev/null +++ b/u3lo36eq/eval_video/15_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3efd5b9c2b7508acddc5a6f0dcbc08a160cc8086197dc6d09621b7e5ff6e4af5 +size 1435091 diff --git a/u3lo36eq/eval_video/16_eval.mp4 b/u3lo36eq/eval_video/16_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8e22e4ded2cb92d774a34ea921b29968a4e55e77 --- /dev/null +++ b/u3lo36eq/eval_video/16_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a29e67cd0c3e166bf43e22739fee1f09eba1a8a562eb67fbd669aa976b607bbe +size 1499720 diff --git a/u3lo36eq/eval_video/17_eval.mp4 b/u3lo36eq/eval_video/17_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..592a5adfde09114777145bf90c0ce89315516f33 --- /dev/null +++ b/u3lo36eq/eval_video/17_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f5ca32b8d4315bd96189e8b5c0c9c42fa14207768d9deacd867cdbbcd94c15d +size 1682243 diff --git a/u3lo36eq/eval_video/18_eval.mp4 b/u3lo36eq/eval_video/18_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..afaba3f340116609ae8c043c0974c1e5f694449a --- /dev/null +++ b/u3lo36eq/eval_video/18_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe8dc0d3e79f58b5302a3c322b3e34f0ea72380aee26862d037cf7f1ac15603d +size 1408117 diff --git a/u3lo36eq/eval_video/19_eval.mp4 b/u3lo36eq/eval_video/19_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c1b5f2a2a8802919ddc939e09a3e7fab0fdcf1aa --- /dev/null +++ b/u3lo36eq/eval_video/19_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98200c8b3f44031bae085047c3976b35accd628515cc49e4bbd40a135554d5d4 +size 1176985 diff --git a/u3lo36eq/eval_video/1_eval.mp4 b/u3lo36eq/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e63adc482c967815bcbfa06143a058a30b7a8b42 --- /dev/null +++ b/u3lo36eq/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f85e4baf2746d94aa0a7c8b7677ae095e9b50b375ccd51cd7d39fddf5749bbf5 +size 1069639 diff --git a/u3lo36eq/eval_video/2_eval.mp4 b/u3lo36eq/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68abd28f925781e77089b969c4e5a2b64ae080b8 --- /dev/null +++ b/u3lo36eq/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:340d87e376b6dfa888fba755c925d901d2e44ae7f004557ae9589d089964e468 +size 1075858 diff --git a/u3lo36eq/eval_video/3_eval.mp4 b/u3lo36eq/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ebc37efcb4d972972ee707a62596834f5aec6dac --- /dev/null +++ b/u3lo36eq/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a45dc41fdb9f639ccef45d7a2e71ae53d23d26ec242ebc388da02b73366f5df3 +size 1155963 diff --git a/u3lo36eq/eval_video/4_eval.mp4 b/u3lo36eq/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..31c3416dabb25562340049957e091f300c6930f2 --- /dev/null +++ b/u3lo36eq/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edd1b447fab3432974b48be99a57b4dc183cd7a1aef9bccc0f20e680f96b6d2c +size 1241795 diff --git a/u3lo36eq/eval_video/5_eval.mp4 b/u3lo36eq/eval_video/5_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0a110c66eeaab03f08452095c3e9f8c42bc77f2d --- /dev/null +++ b/u3lo36eq/eval_video/5_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c150bb925f103adcede99be565e6a2271f8d73019cdeeff55855ffeacbe1952e +size 1202637 diff --git a/u3lo36eq/eval_video/6_eval.mp4 b/u3lo36eq/eval_video/6_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..372e0c2eb21c0c49c2ffb8fbeb56db6cf5bd41b3 --- /dev/null +++ b/u3lo36eq/eval_video/6_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:438f8d27cd8ba8c28d6a922dd0610eb5627087e78728af8b2dbb7d5e9e0775c2 +size 1250346 diff --git a/u3lo36eq/eval_video/7_eval.mp4 b/u3lo36eq/eval_video/7_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d8daca8d8cbd4883f5ff509c172ffb3bacae752b --- /dev/null +++ b/u3lo36eq/eval_video/7_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d015da1851d414bb66284e74bb9c0ca9bef27dd1ca7bd3169f2b998741a08d99 +size 1203795 diff --git a/u3lo36eq/eval_video/8_eval.mp4 b/u3lo36eq/eval_video/8_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b3fcb545073977d445e718885dd5f27621585468 --- /dev/null +++ b/u3lo36eq/eval_video/8_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d16e439b84d1b213c76a32b6206b6b6d2b797bae4880b2182e5b6179572d8cf +size 1283069 diff --git a/u3lo36eq/eval_video/9_eval.mp4 b/u3lo36eq/eval_video/9_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..42e26c4ea8052a2a1443527a92d0dfb465d57802 --- /dev/null +++ b/u3lo36eq/eval_video/9_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d74f7b5c31b1fe3c2190fc0a42914b3831fceb88147e73d6a09d33bd35a4f072 +size 1159824 diff --git a/u3lo36eq/tb/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 b/u3lo36eq/tb/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 new file mode 100644 index 0000000000000000000000000000000000000000..f3ac4b1db7f0f390034623b6e1db158a2ecfd2ad --- /dev/null +++ b/u3lo36eq/tb/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:028910711e02d255b026551d67eae3599853c2e60677fba55a415beb3ca4bbd1 +size 22764 diff --git a/u3lo36eq/wandb/debug-internal.log b/u3lo36eq/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..983a2d3bab131751666b486e9c8f490ca1b52573 --- /dev/null +++ b/u3lo36eq/wandb/debug-internal.log @@ -0,0 +1,2258 @@ +2025-01-18 19:08:53,484 INFO StreamThr :2557979 [internal.py:wandb_internal():85] W&B internal server running at pid: 2557979, started at: 2025-01-18 19:08:53.483861 +2025-01-18 19:08:53,485 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status +2025-01-18 19:08:53,496 INFO WriterThread:2557979 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/run-u3lo36eq.wandb +2025-01-18 19:08:53,496 DEBUG SenderThread:2557979 [sender.py:send():379] send: header +2025-01-18 19:08:53,503 DEBUG SenderThread:2557979 [sender.py:send():379] send: run +2025-01-18 19:08:53,707 INFO SenderThread:2557979 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files +2025-01-18 19:08:53,707 INFO SenderThread:2557979 [sender.py:_start_run_threads():1188] run started: u3lo36eq with start time 1737245333.483906 +2025-01-18 19:08:53,731 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: check_version +2025-01-18 19:08:53,731 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: check_version +2025-01-18 19:08:53,808 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: run_start +2025-01-18 19:08:53,813 DEBUG HandlerThread:2557979 [system_info.py:__init__():26] System info init +2025-01-18 19:08:53,813 DEBUG HandlerThread:2557979 [system_info.py:__init__():41] System info init done +2025-01-18 19:08:53,813 INFO HandlerThread:2557979 [system_monitor.py:start():194] Starting system monitor +2025-01-18 19:08:53,813 INFO SystemMonitor:2557979 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-18 19:08:53,814 INFO HandlerThread:2557979 [system_monitor.py:probe():214] Collecting system info +2025-01-18 19:08:53,814 INFO SystemMonitor:2557979 [interfaces.py:start():188] Started cpu monitoring +2025-01-18 19:08:53,815 INFO SystemMonitor:2557979 [interfaces.py:start():188] Started disk monitoring +2025-01-18 19:08:53,815 INFO SystemMonitor:2557979 [interfaces.py:start():188] Started gpu monitoring +2025-01-18 19:08:53,816 INFO SystemMonitor:2557979 [interfaces.py:start():188] Started memory monitoring +2025-01-18 19:08:53,816 INFO SystemMonitor:2557979 [interfaces.py:start():188] Started network monitoring +2025-01-18 19:08:53,827 DEBUG HandlerThread:2557979 [system_info.py:probe():152] Probing system +2025-01-18 19:08:53,829 DEBUG HandlerThread:2557979 [system_info.py:_probe_git():137] Probing git +2025-01-18 19:08:53,833 DEBUG HandlerThread:2557979 [system_info.py:_probe_git():145] Probing git done +2025-01-18 19:08:53,833 DEBUG HandlerThread:2557979 [system_info.py:probe():200] Probing system done +2025-01-18 19:08:53,833 DEBUG HandlerThread:2557979 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-19T00:08:53.827800', 'startedAt': '2025-01-19T00:08:53.469225', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3383.22575, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4543.658, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 524.6030311584473}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-18 19:08:53,833 INFO HandlerThread:2557979 [system_monitor.py:probe():224] Finished collecting system info +2025-01-18 19:08:53,834 INFO HandlerThread:2557979 [system_monitor.py:probe():227] Publishing system info +2025-01-18 19:08:53,834 INFO HandlerThread:2557979 [system_monitor.py:probe():229] Finished publishing system info +2025-01-18 19:08:53,837 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:08:53,837 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-18 19:08:54,075 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: python_packages +2025-01-18 19:08:54,076 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: python_packages +2025-01-18 19:08:54,087 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/vbgd9pni-wandb-metadata.json +2025-01-18 19:08:54,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:08:54,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:08:54,117 INFO HandlerThread:2557979 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/tb" + save: true +} +control { + relay_id: "60f290ddaf4c43f8ad9368cf744d939e" +} +_info { + stream_id: "u3lo36eq" +} + +2025-01-18 19:08:54,117 DEBUG HandlerThread:2557979 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-18 19:08:54,118 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:08:54,228 DEBUG SenderThread:2557979 [sender.py:send():379] send: telemetry +2025-01-18 19:08:54,228 DEBUG SenderThread:2557979 [sender.py:send():379] send: tbrecord +2025-01-18 19:08:54,229 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:08:54,229 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 with policy live +2025-01-18 19:08:54,406 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/mp593bay-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:08:54,710 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-metadata.json +2025-01-18 19:08:54,710 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:08:54,710 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/requirements.txt +2025-01-18 19:08:54,710 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:08:55,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:08:56,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:08:56,708 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:08:57,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:08:58,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:08:58,121 DEBUG SenderThread:2557979 [sender.py:send():379] send: config +2025-01-18 19:08:59,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:08:59,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:00,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:00,709 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:01,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:02,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:03,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:04,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:04,710 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:05,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:05,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:06,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:06,711 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:07,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:08,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:09,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:09:09,076 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:09:09,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:10,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:10,152 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:11,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:12,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:13,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:14,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:15,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:16,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:16,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:17,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:18,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:19,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:20,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:20,714 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:21,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:21,177 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:22,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:22,714 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:23,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:24,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:09:24,076 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:09:24,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:24,715 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:25,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:26,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:26,191 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:26,715 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/config.yaml +2025-01-18 19:09:27,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:28,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:29,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:30,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:30,716 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:31,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:32,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:32,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:33,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:34,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:35,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:36,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:37,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:37,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:38,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:39,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:09:39,076 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:09:39,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:40,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:41,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:42,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:42,144 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:43,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:44,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:45,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:46,083 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:47,083 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:47,496 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:48,083 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:48,542 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:09:48,542 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:09:48,542 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_f2d24d1f27d7648055dc.mp4 with policy now +2025-01-18 19:09:48,543 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:09:48,543 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:09:48,543 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:09:48,720 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:48,720 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/0_eval_0_f2d24d1f27d7648055dc.mp4 +2025-01-18 19:09:48,720 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:09:48,721 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media +2025-01-18 19:09:48,721 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:09:48,721 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos +2025-01-18 19:09:48,815 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/ruzqdu7e-media/videos/eval/0_eval_0_f2d24d1f27d7648055dc.mp4 +2025-01-18 19:09:49,083 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:49,100 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/9gurev3a-media/videos/eval/0_eval_0_f2d24d1f27d7648055dc.mp4 +2025-01-18 19:09:50,083 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:50,721 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:51,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:52,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:52,993 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:53,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:53,816 DEBUG SystemMonitor:2557979 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-18 19:09:53,817 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:09:54,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:09:54,077 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:09:54,125 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:54,722 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:55,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:56,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:56,723 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:57,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:58,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:58,706 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:59,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:00,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:01,085 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:02,085 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:03,085 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:04,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:04,085 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:05,085 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:06,085 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:06,726 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:10:06,972 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/8oy5cb5r-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:10:07,085 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:08,085 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:08,727 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:09,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:10:09,077 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:10:09,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:09,223 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:10,105 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:11,086 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:12,086 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:12,728 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:13,086 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:14,086 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:15,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:15,086 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:16,087 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:17,087 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:18,087 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:19,087 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:20,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:20,087 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:21,087 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:22,087 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:23,087 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:23,826 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:10:24,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:10:24,077 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:10:24,087 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:24,731 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:25,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:25,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:26,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:27,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:28,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:29,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:30,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:30,733 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:31,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:31,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:32,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:33,089 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:34,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:34,734 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:34,951 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:10:34,951 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:10:34,951 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f85e4baf2746d94aa0a7.mp4 with policy now +2025-01-18 19:10:34,952 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:10:34,952 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:10:34,952 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:10:35,092 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:35,215 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/511xv1rb-media/videos/eval/1_eval_1_f85e4baf2746d94aa0a7.mp4 +2025-01-18 19:10:35,735 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:10:35,735 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/1_eval_1_f85e4baf2746d94aa0a7.mp4 +2025-01-18 19:10:35,735 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:10:36,051 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/cgbbm1q3-media/videos/eval/1_eval_1_f85e4baf2746d94aa0a7.mp4 +2025-01-18 19:10:36,093 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:36,735 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:36,968 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:37,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:37,736 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:38,093 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:39,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:10:39,080 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:10:39,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:40,093 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:40,736 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:41,133 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:42,093 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:42,134 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:42,737 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:43,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:44,093 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:44,738 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:10:45,093 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:45,518 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:10:45,518 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:10:45,519 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:10:45,738 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:10:46,093 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:46,738 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:47,093 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:47,601 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:48,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:49,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:50,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:50,739 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:51,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:52,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:52,740 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:53,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:53,317 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:53,818 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:10:54,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:10:54,078 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:10:54,121 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:55,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:56,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:57,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:58,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:59,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:59,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:00,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:01,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:02,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:03,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:04,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:05,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:05,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:06,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:07,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:08,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:09,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:11:09,077 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:11:09,121 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:10,096 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:10,200 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:11,096 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:12,096 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:13,098 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:13,866 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:11:13,866 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_340d87e376b6dfa888fb.mp4 with policy now +2025-01-18 19:11:13,869 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:11:13,869 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:11:13,870 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:11:13,871 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:11:14,098 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:14,126 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/vmi8qezn-media/videos/eval/2_eval_2_340d87e376b6dfa888fb.mp4 +2025-01-18 19:11:14,747 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:11:14,747 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:11:14,747 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/2_eval_2_340d87e376b6dfa888fb.mp4 +2025-01-18 19:11:14,748 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:11:15,063 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/cnt70odx-media/videos/eval/2_eval_2_340d87e376b6dfa888fb.mp4 +2025-01-18 19:11:15,103 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:15,747 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:11:16,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:16,109 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:17,103 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:18,106 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:18,748 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:11:19,110 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:20,110 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:21,108 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:21,209 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:22,114 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:22,750 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:11:23,114 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:23,818 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:11:24,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:11:24,078 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:11:24,125 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:25,112 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:26,111 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:26,215 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:26,751 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:11:26,998 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/k1aae7z8-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:11:27,113 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:27,351 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:11:27,351 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:11:27,352 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:11:27,751 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:11:28,116 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:28,751 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:11:29,116 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:30,118 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:31,116 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:31,876 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:32,115 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:32,753 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:11:33,115 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:34,116 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:34,753 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:11:35,116 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:36,116 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:37,083 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:37,116 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:38,116 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:39,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:11:39,078 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:11:39,121 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:40,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:41,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:42,116 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:42,207 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:43,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:44,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:45,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:46,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:47,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:48,083 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:48,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:49,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:50,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:51,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:52,118 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:53,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:53,118 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:53,819 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:11:54,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:11:54,078 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:11:54,121 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:55,118 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:56,118 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:57,118 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:58,120 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:58,378 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:58,760 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:11:58,984 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:11:58,984 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a45dc41fdb9f639ccef4.mp4 with policy now +2025-01-18 19:11:58,984 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:11:58,985 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:11:58,985 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:11:58,986 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:11:59,120 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:59,265 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/vzvvmmix-media/videos/eval/3_eval_3_a45dc41fdb9f639ccef4.mp4 +2025-01-18 19:11:59,761 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:11:59,761 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/3_eval_3_a45dc41fdb9f639ccef4.mp4 +2025-01-18 19:11:59,761 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:12:00,107 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/63kl3bky-media/videos/eval/3_eval_3_a45dc41fdb9f639ccef4.mp4 +2025-01-18 19:12:00,125 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:00,761 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:01,125 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:01,761 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:02,137 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:03,131 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:03,565 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:04,129 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:04,762 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:05,136 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:06,129 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:06,763 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:07,130 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:08,131 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:09,083 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:12:09,083 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:12:09,131 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:09,221 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:10,134 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:11,133 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:12,133 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:13,139 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:14,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:15,090 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:15,138 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:16,139 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:17,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:18,138 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:18,414 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:12:18,414 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:12:18,414 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:12:18,766 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:12:18,766 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:18,766 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:12:19,137 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:20,091 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:20,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:20,767 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:21,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:22,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:22,767 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:23,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:23,819 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:12:24,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:12:24,080 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:12:24,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:24,768 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:25,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:25,152 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:26,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:27,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:28,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:28,770 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:29,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:30,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:31,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:31,141 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:32,141 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:33,141 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:34,141 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:35,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:36,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:36,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:37,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:38,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:39,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:12:39,081 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:12:39,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:40,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:41,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:41,180 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:42,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:43,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:44,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:45,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:46,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:47,089 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:47,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:48,144 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:49,144 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:50,144 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:51,149 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:51,625 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:12:51,625 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:12:51,625 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_edd1b447fab3432974b4.mp4 with policy now +2025-01-18 19:12:51,634 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:12:51,635 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:12:51,636 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:12:51,777 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:12:51,777 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/4_eval_4_edd1b447fab3432974b4.mp4 +2025-01-18 19:12:51,778 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:12:51,896 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/mlhtw6xu-media/videos/eval/4_eval_4_edd1b447fab3432974b4.mp4 +2025-01-18 19:12:52,162 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:52,309 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/z7dphjvn-media/videos/eval/4_eval_4_edd1b447fab3432974b4.mp4 +2025-01-18 19:12:52,637 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:52,778 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:53,154 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:53,820 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:12:54,112 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:12:54,112 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:12:54,156 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:55,160 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:56,155 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:56,779 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:57,161 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:58,089 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:58,152 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:59,156 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:00,159 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:00,780 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:01,164 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:01,466 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:13:01,467 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:13:01,467 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:13:01,780 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:13:01,780 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:13:02,042 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/mm5z1mqe-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:13:02,163 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:02,781 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:03,175 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:03,225 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:04,171 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:04,781 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:05,166 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:06,171 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:06,782 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:07,167 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:08,168 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:08,662 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:08,783 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:09,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:13:09,082 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:13:09,169 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:10,168 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:11,168 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:12,169 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:13,169 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:14,090 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:14,169 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:15,169 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:16,169 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:17,169 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:18,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:19,090 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:19,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:20,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:20,786 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:21,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:22,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:23,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:23,820 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:13:24,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:13:24,082 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:13:24,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:24,236 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:25,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:26,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:26,788 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:27,171 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:28,171 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:29,171 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:30,091 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:30,171 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:31,172 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:32,172 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:33,172 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:34,209 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:34,791 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:34,926 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:13:34,926 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_c150bb925f103adcede9.mp4 with policy now +2025-01-18 19:13:34,946 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:13:34,947 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:13:34,947 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:13:34,947 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:13:35,179 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:35,200 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/agr22d06-media/videos/eval/5_eval_5_c150bb925f103adcede9.mp4 +2025-01-18 19:13:35,791 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:13:35,791 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/5_eval_5_c150bb925f103adcede9.mp4 +2025-01-18 19:13:35,791 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:13:35,994 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:36,112 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/qh0l4mdb-media/videos/eval/5_eval_5_c150bb925f103adcede9.mp4 +2025-01-18 19:13:36,184 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:36,791 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:37,188 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:37,792 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:38,181 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:39,091 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:13:39,091 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:13:39,181 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:40,182 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:40,793 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:41,182 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:41,225 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:42,182 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:42,793 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:43,181 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:44,191 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:44,794 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:13:45,188 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:45,521 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:13:45,521 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:13:45,522 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:13:45,794 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:13:45,794 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:13:46,199 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:46,795 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:46,889 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:47,192 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:48,198 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:49,190 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:50,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:50,796 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:51,189 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:52,168 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:52,189 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:52,796 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:53,189 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:53,820 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:13:54,085 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:13:54,085 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:13:54,190 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:55,190 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:56,190 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:57,191 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:58,167 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:58,191 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:59,191 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:00,195 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:01,193 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:02,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:03,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:03,545 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:04,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:04,800 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:05,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:06,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:07,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:08,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:09,086 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:14:09,086 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:14:09,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:09,261 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:10,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:10,802 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:11,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:12,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:13,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:14,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:14,307 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:15,195 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:16,195 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:16,804 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:17,195 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:18,203 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:18,805 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:19,093 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:14:19,093 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_438f8d27cd8ba8c28d6a.mp4 with policy now +2025-01-18 19:14:19,101 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:14:19,102 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:14:19,102 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:14:19,102 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:14:19,198 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:19,364 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/o61o0xqm-media/videos/eval/6_eval_6_438f8d27cd8ba8c28d6a.mp4 +2025-01-18 19:14:19,805 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:14:19,805 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/6_eval_6_438f8d27cd8ba8c28d6a.mp4 +2025-01-18 19:14:19,805 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:14:20,111 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/p9fnoe43-media/videos/eval/6_eval_6_438f8d27cd8ba8c28d6a.mp4 +2025-01-18 19:14:20,157 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:20,216 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:20,805 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:21,212 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:22,204 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:23,221 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:23,821 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:14:24,120 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:14:24,120 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:14:24,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:24,807 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:25,202 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:25,212 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:26,213 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:26,807 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:27,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:28,215 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:29,211 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:30,241 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:30,242 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:30,576 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:14:30,577 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:14:30,577 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:14:30,808 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:14:30,809 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:14:31,054 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/vwd1mvin-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:14:31,217 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:32,217 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:32,809 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:33,219 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:34,217 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:35,217 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:35,558 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:36,217 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:36,811 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:37,217 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:38,217 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:38,811 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:39,086 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:14:39,086 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:14:39,217 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:40,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:41,180 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:41,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:42,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:43,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:44,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:44,813 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:45,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:46,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:46,737 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:46,814 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:47,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:48,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:49,219 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:50,219 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:51,219 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:52,098 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:52,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:53,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:53,821 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:14:54,086 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:14:54,087 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:14:54,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:54,817 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:55,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:56,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:57,167 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:57,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:58,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:59,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:00,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:01,221 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:02,225 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:02,347 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:02,397 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:15:02,397 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_d015da1851d414bb6628.mp4 with policy now +2025-01-18 19:15:02,419 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:15:02,420 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:15:02,420 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:15:02,421 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:15:02,668 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/slv61nk6-media/videos/eval/7_eval_7_d015da1851d414bb6628.mp4 +2025-01-18 19:15:02,819 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:02,819 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:15:02,819 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/7_eval_7_d015da1851d414bb6628.mp4 +2025-01-18 19:15:02,820 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:15:03,122 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/dwykm6q5-media/videos/eval/7_eval_7_d015da1851d414bb6628.mp4 +2025-01-18 19:15:03,257 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:03,819 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:04,242 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:04,820 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:05,268 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:06,252 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:07,247 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:08,248 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:08,249 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:08,821 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:09,100 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:15:09,100 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:15:09,247 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:10,263 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:10,822 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:11,265 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:12,275 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:13,266 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:14,126 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:14,268 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:15,273 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:16,268 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:17,268 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:18,271 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:19,272 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:20,114 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:20,273 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:21,270 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:22,286 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:22,642 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:15:22,643 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:15:22,644 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:15:22,825 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:15:22,826 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:22,826 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:15:23,279 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:23,822 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:15:24,093 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:15:24,094 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:15:24,282 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:24,826 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:25,168 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:25,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:26,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:27,292 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:28,280 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:28,827 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:29,295 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:30,252 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:30,281 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:30,828 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:31,281 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:32,282 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:32,829 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:33,281 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:34,282 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:35,282 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:36,108 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:36,282 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:37,282 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:38,282 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:39,091 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:15:39,091 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:15:39,282 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:40,282 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:41,229 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:41,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:42,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:43,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:44,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:45,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:46,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:47,108 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:47,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:48,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:49,284 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:50,284 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:51,284 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:52,109 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:52,284 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:53,284 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:53,822 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:15:54,091 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:15:54,091 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:15:54,284 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:55,285 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:56,285 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:57,290 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:57,843 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:58,291 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:58,576 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:15:58,576 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_9d16e439b84d1b213c76.mp4 with policy now +2025-01-18 19:15:58,582 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:15:58,582 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:15:58,582 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:15:58,582 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:15:58,848 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:58,848 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:15:58,848 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/8_eval_8_9d16e439b84d1b213c76.mp4 +2025-01-18 19:15:58,848 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:15:59,185 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/2v1k81xu-media/videos/eval/8_eval_8_9d16e439b84d1b213c76.mp4 +2025-01-18 19:15:59,298 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:59,472 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/r20sy1rb-media/videos/eval/8_eval_8_9d16e439b84d1b213c76.mp4 +2025-01-18 19:15:59,848 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:00,305 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:00,848 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:01,302 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:01,849 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:02,324 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:02,849 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:03,181 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:03,328 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:04,330 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:05,308 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:06,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:06,850 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:07,327 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:08,237 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:08,323 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:09,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:16:09,117 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:16:09,331 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:10,329 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:11,326 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:11,705 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:16:11,705 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:16:11,705 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:16:11,852 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:16:11,852 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:16:12,094 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/a6x8cl2s-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:16:12,347 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:12,852 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:13,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:13,397 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:14,350 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:14,853 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:15,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:16,332 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:16,854 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:17,333 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:18,333 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:18,799 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:19,333 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:20,333 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:21,333 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:22,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:23,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:23,823 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:16:23,824 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:24,114 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:16:24,114 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:16:24,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:25,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:26,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:27,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:28,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:29,126 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:29,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:30,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:31,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:32,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:33,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:34,127 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:34,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:35,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:36,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:37,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:38,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:39,114 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:16:39,114 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:16:39,200 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:39,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:40,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:41,336 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:42,336 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:43,341 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:44,099 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:16:44,099 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_d74f7b5c31b1fe3c2190.mp4 with policy now +2025-01-18 19:16:44,104 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:16:44,105 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:16:44,105 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:16:44,105 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:16:44,344 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:44,359 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/is25lsai-media/videos/eval/9_eval_9_d74f7b5c31b1fe3c2190.mp4 +2025-01-18 19:16:44,863 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:44,863 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:16:44,863 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/9_eval_9_d74f7b5c31b1fe3c2190.mp4 +2025-01-18 19:16:44,864 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:16:45,174 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/drx1gn77-media/videos/eval/9_eval_9_d74f7b5c31b1fe3c2190.mp4 +2025-01-18 19:16:45,190 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:45,360 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:45,863 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:46,353 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:47,377 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:48,353 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:48,864 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:49,354 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:50,363 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:50,695 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:51,360 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:52,357 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:52,866 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:53,365 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:53,823 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:16:54,129 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:16:54,129 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:16:54,366 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:55,386 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:56,238 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:56,379 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:57,376 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:58,422 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:59,419 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:00,424 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:01,419 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:02,150 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:02,419 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:03,426 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:04,422 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:05,423 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:06,426 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:07,153 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:07,448 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:08,449 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:09,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:17:09,143 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:17:09,450 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:10,446 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:11,473 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:11,778 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:17:11,778 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:17:11,778 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:17:11,872 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:17:11,872 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:17:12,354 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:12,478 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:12,873 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:13,470 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:14,471 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:14,874 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:15,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:16,473 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:16,874 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:17,401 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:17,485 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:18,466 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:18,875 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:19,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:20,466 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:20,876 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:21,466 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:22,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:23,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:23,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:23,824 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:17:24,127 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:17:24,128 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:17:24,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:25,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:26,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:27,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:28,203 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:28,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:29,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:30,468 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:31,468 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:32,468 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:33,468 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:34,139 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:34,468 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:35,468 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:36,468 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:37,468 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:38,468 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:38,882 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:39,127 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:17:39,128 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:17:39,240 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:39,469 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:40,469 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:41,469 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:42,470 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:43,470 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:44,470 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:45,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:45,477 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:46,481 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:46,681 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:17:46,681 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_741429cbea5a900591e5.mp4 with policy now +2025-01-18 19:17:46,681 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:17:46,682 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:17:46,682 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:17:46,683 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:17:46,885 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:46,885 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:17:46,885 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/10_eval_10_741429cbea5a900591e5.mp4 +2025-01-18 19:17:46,885 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:17:46,982 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/cyxwk091-media/videos/eval/10_eval_10_741429cbea5a900591e5.mp4 +2025-01-18 19:17:47,277 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/tlr2jk1a-media/videos/eval/10_eval_10_741429cbea5a900591e5.mp4 +2025-01-18 19:17:47,484 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:47,885 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:48,511 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:48,886 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:49,496 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:50,165 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:50,480 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:50,886 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:51,484 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:52,507 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:53,482 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:53,834 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:17:54,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:17:54,142 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:17:54,497 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:54,887 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:55,216 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:55,502 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:56,507 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:57,509 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:58,523 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:59,524 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:00,533 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:01,166 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:01,529 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:02,544 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:03,550 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:04,567 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:05,564 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:06,547 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:07,161 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:07,547 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:08,588 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:09,134 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:18:09,135 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:18:09,560 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:10,559 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:11,577 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:12,208 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:12,560 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:13,570 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:13,860 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:18:13,861 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:18:13,861 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:18:13,894 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:18:13,894 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:18:14,135 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/smvssqfe-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:18:14,621 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:14,894 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:18:15,575 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:16,576 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:16,895 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:18:17,563 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:17,609 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:18,581 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:18,896 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:18:19,582 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:20,582 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:21,582 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:22,582 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:23,155 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:23,582 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:23,824 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:18:24,129 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:18:24,130 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:18:24,582 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:25,582 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:26,582 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:27,583 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:28,267 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:28,583 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:29,583 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:30,583 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:31,583 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:32,583 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:33,584 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:34,155 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:34,584 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:35,584 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:36,584 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:37,584 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:38,584 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:39,129 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:18:39,130 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:18:39,263 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:39,584 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:40,585 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:40,903 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:18:41,588 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:41,757 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:18:41,757 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_5d5f6eccd1ac8a9641c2.mp4 with policy now +2025-01-18 19:18:41,759 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:18:41,760 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:18:41,760 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:18:41,760 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:18:41,904 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:18:41,904 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/11_eval_11_5d5f6eccd1ac8a9641c2.mp4 +2025-01-18 19:18:41,904 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:18:42,014 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/ynl7vygw-media/videos/eval/11_eval_11_5d5f6eccd1ac8a9641c2.mp4 +2025-01-18 19:18:42,310 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/dc8azret-media/videos/eval/11_eval_11_5d5f6eccd1ac8a9641c2.mp4 +2025-01-18 19:18:42,592 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:42,904 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:18:43,611 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:43,904 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:18:44,591 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:45,203 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:45,606 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:46,603 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:46,905 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:18:47,616 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:48,623 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:49,612 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:50,616 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:50,907 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:18:51,163 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:51,617 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:52,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:53,632 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:53,825 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:18:54,162 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:18:54,163 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:18:54,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:55,638 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:56,230 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:56,651 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:56,909 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:18:56,909 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:18:56,909 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:18:56,910 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:18:56,910 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:18:57,632 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:57,910 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:18:58,639 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:58,910 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:18:59,645 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:00,653 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:01,320 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:01,628 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:02,628 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:02,912 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:03,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:04,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:05,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:06,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:06,813 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:06,914 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:07,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:08,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:09,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:19:09,145 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:19:09,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:10,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:11,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:12,271 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:12,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:13,630 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:14,630 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:15,630 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:16,630 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:17,630 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:18,158 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:18,630 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:19,630 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:20,631 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:21,631 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:22,634 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:23,159 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:23,635 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:23,825 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:19:24,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:19:24,145 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:19:24,633 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:24,920 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:25,634 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:26,636 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:27,635 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:28,179 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:28,677 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:28,921 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:29,468 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:19:29,468 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_9c363b6170975b374255.mp4 with policy now +2025-01-18 19:19:29,491 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:19:29,491 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:19:29,492 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:19:29,492 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:19:29,644 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:29,745 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/df1eu0ym-media/videos/eval/12_eval_12_9c363b6170975b374255.mp4 +2025-01-18 19:19:29,922 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:19:29,922 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/12_eval_12_9c363b6170975b374255.mp4 +2025-01-18 19:19:29,922 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:19:30,242 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/pz1b8jun-media/videos/eval/12_eval_12_9c363b6170975b374255.mp4 +2025-01-18 19:19:30,655 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:30,922 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:31,653 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:31,923 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:32,670 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:33,195 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:33,668 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:34,722 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:34,924 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:35,678 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:36,666 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:36,925 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:37,680 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:38,417 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:38,695 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:39,193 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:19:39,194 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:19:39,709 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:39,926 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:19:39,966 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:19:39,966 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:19:39,966 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:19:40,712 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:40,926 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:40,926 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:19:41,708 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:42,732 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:43,477 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:43,700 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:44,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:44,927 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:45,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:46,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:47,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:48,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:49,167 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:49,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:50,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:51,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:52,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:53,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:53,826 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:19:54,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:19:54,145 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:19:54,221 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:54,699 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:55,699 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:56,699 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:57,699 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:58,699 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:59,700 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:00,168 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:00,700 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:01,701 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:02,701 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:03,701 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:04,701 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:05,701 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:06,168 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:06,701 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:07,702 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:08,702 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:09,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:20:09,145 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:20:09,702 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:10,702 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:11,667 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:11,731 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:12,396 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:20:12,396 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_c552251a3cda7642b754.mp4 with policy now +2025-01-18 19:20:12,410 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:20:12,411 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:20:12,411 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:20:12,411 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:20:12,705 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/nn254u81-media/videos/eval/13_eval_13_c552251a3cda7642b754.mp4 +2025-01-18 19:20:12,712 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:12,938 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:20:12,943 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/13_eval_13_c552251a3cda7642b754.mp4 +2025-01-18 19:20:12,943 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:20:13,272 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/4bkggix9-media/videos/eval/13_eval_13_c552251a3cda7642b754.mp4 +2025-01-18 19:20:13,723 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:13,939 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:20:14,725 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:15,765 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:16,754 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:17,153 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:17,789 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:17,940 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:20:18,778 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:19,742 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:20,755 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:21,786 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:21,942 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:20:22,433 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:22,787 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:23,767 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:23,826 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:20:24,166 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:20:24,166 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:20:24,755 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:25,764 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:26,803 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:27,784 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:28,263 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:28,767 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:29,780 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:30,809 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:31,763 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:32,780 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:33,802 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:34,179 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:34,815 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:35,820 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:36,803 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:37,812 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:38,793 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:39,179 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:39,183 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:20:39,183 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:20:39,815 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:40,848 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:41,812 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:42,841 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:43,817 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:44,191 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:44,821 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:45,833 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:46,840 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:47,824 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:48,836 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:49,192 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:49,830 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:50,909 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:51,891 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:52,865 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:53,827 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:20:53,876 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:54,190 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:20:54,191 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:20:54,279 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:54,895 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:55,896 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:56,893 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:57,890 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:58,888 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:59,898 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:00,210 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:00,895 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:01,908 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:02,922 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:03,919 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:04,956 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:05,924 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:06,202 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:06,927 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:07,922 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:08,943 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:09,213 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:21:09,214 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:21:09,945 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:10,950 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:11,380 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:11,932 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:12,946 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:13,938 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:14,934 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:15,928 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:16,938 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:17,213 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:17,986 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:18,994 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:20,011 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:20,982 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:21,999 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:22,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:23,011 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:23,827 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:21:24,012 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:24,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:21:24,170 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:21:25,002 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:26,006 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:26,965 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:21:27,021 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:27,090 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:21:27,090 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:21:27,091 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:21:27,217 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/8z9t98wv-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:21:27,777 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:27,965 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:21:27,965 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:21:28,018 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:29,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:29,966 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:21:30,029 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:31,031 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:32,035 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:32,986 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:33,034 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:33,967 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:21:34,036 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:35,059 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:36,031 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:37,065 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:38,062 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:38,239 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:39,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:39,201 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:21:39,202 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:21:40,036 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:41,036 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:41,970 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:21:42,036 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:43,036 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:43,349 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:44,036 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:45,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:46,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:47,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:48,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:49,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:49,215 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:50,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:51,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:52,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:53,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:53,828 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:21:54,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:54,168 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:21:54,168 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:21:54,280 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:55,038 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:56,038 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:57,038 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:58,038 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:59,038 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:00,038 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:00,216 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:01,039 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:02,039 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:03,039 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:04,039 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:05,039 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:05,965 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:06,040 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:07,045 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:07,306 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:22:07,306 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_58248ea2be38c14379cd.mp4 with policy now +2025-01-18 19:22:07,310 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:22:07,310 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:22:07,310 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:22:07,311 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:22:07,616 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/04mdop80-media/videos/eval/14_eval_14_58248ea2be38c14379cd.mp4 +2025-01-18 19:22:07,980 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:22:07,980 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:22:07,980 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/14_eval_14_58248ea2be38c14379cd.mp4 +2025-01-18 19:22:07,980 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:22:08,044 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:08,337 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/g1xeiq7w-media/videos/eval/14_eval_14_58248ea2be38c14379cd.mp4 +2025-01-18 19:22:09,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:09,176 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:22:09,176 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:22:09,980 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:22:10,066 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:10,977 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:11,074 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:12,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:13,074 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:13,982 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:22:14,073 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:15,072 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:15,983 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:22:16,096 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:16,266 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:17,091 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:18,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:19,089 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:19,984 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:22:20,098 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:20,156 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:22:20,156 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:22:20,157 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:22:20,984 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:22:20,985 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:22:21,119 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:21,985 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:22:22,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:22,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:23,126 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:23,829 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:22:23,986 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:22:24,118 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:24,181 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:22:24,181 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:22:25,160 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:25,986 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:22:26,130 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:27,132 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:27,413 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:28,154 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:29,153 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:30,162 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:31,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:31,989 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:22:32,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:32,570 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:33,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:33,989 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:22:34,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:35,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:36,144 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:37,144 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:38,144 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:38,264 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:39,144 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:39,173 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:22:39,173 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:22:40,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:41,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:42,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:43,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:43,264 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:44,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:45,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:46,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:47,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:48,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:48,265 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:49,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:50,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:51,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:52,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:53,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:53,829 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:22:53,830 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:54,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:54,174 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:22:54,174 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:22:55,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:56,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:57,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:58,157 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:59,074 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:59,167 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:59,194 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:22:59,194 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_3efd5b9c2b7508acddc5.mp4 with policy now +2025-01-18 19:22:59,237 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:22:59,238 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:22:59,238 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:22:59,240 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:22:59,498 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/abunlzd2-media/videos/eval/15_eval_15_3efd5b9c2b7508acddc5.mp4 +2025-01-18 19:22:59,999 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:00,000 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:23:00,000 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/15_eval_15_3efd5b9c2b7508acddc5.mp4 +2025-01-18 19:23:00,000 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:23:00,210 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:00,379 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/zvjhtqjt-media/videos/eval/15_eval_15_3efd5b9c2b7508acddc5.mp4 +2025-01-18 19:23:01,166 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:02,000 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:02,209 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:03,174 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:04,189 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:04,311 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:05,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:06,002 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:06,180 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:07,190 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:08,002 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:08,189 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:09,184 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:23:09,185 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:23:09,225 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:09,322 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:10,192 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:11,190 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:12,215 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:13,202 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:14,216 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:14,323 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:15,214 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:16,005 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:23:16,225 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:23:16,226 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:23:16,226 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:23:16,237 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:17,006 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:23:17,214 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:18,006 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:18,274 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:19,286 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:20,008 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:20,044 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:20,261 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:21,278 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:22,009 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:22,316 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:23,292 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:23,829 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:23:24,211 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:23:24,211 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:23:24,295 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:25,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:25,315 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:26,309 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:27,307 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:28,346 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:29,338 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:30,310 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:30,446 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:31,311 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:32,012 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:32,311 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:33,311 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:34,312 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:35,312 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:36,309 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:36,312 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:37,312 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:38,312 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:39,180 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:23:39,180 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:23:39,313 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:40,313 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:41,313 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:41,331 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:42,313 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:43,313 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:44,313 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:45,313 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:46,313 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:47,309 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:47,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:48,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:49,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:50,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:51,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:52,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:52,379 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:53,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:53,830 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:23:54,022 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:54,180 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:23:54,180 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:23:54,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:55,315 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:56,022 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:56,315 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:57,333 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:57,498 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:57,787 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:23:57,787 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_a29e67cd0c3e166bf43e.mp4 with policy now +2025-01-18 19:23:57,805 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:23:57,805 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:23:57,806 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:23:57,806 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:23:58,023 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:58,023 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:23:58,023 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/16_eval_16_a29e67cd0c3e166bf43e.mp4 +2025-01-18 19:23:58,024 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:23:58,097 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/iw0ep7i3-media/videos/eval/16_eval_16_a29e67cd0c3e166bf43e.mp4 +2025-01-18 19:23:58,321 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:58,430 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/v3rfnh24-media/videos/eval/16_eval_16_a29e67cd0c3e166bf43e.mp4 +2025-01-18 19:23:59,332 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:00,024 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:24:00,345 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:01,346 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:02,359 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:03,366 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:03,469 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:04,025 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:24:04,417 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:05,392 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:06,026 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:24:06,404 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:07,389 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:08,411 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:08,758 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:09,211 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:24:09,211 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:24:09,404 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:10,442 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:11,432 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:12,423 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:13,029 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:24:13,279 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/cvqcv4bk-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:24:13,293 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:24:13,293 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:24:13,294 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:24:13,434 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:14,029 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:24:14,029 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:24:14,337 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:14,434 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:15,442 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:16,030 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:24:16,437 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:17,458 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:18,446 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:19,346 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:19,449 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:20,031 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:24:20,462 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:21,458 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:22,456 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:23,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:23,830 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:24:24,216 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:24:24,217 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:24:24,448 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:25,341 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:25,466 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:26,465 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:27,461 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:28,460 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:29,461 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:30,035 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:24:30,461 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:31,330 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:31,461 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:32,461 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:33,462 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:34,462 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:35,462 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:36,330 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:36,462 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:37,462 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:38,463 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:39,201 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:24:39,201 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:24:39,463 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:40,463 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:41,331 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:41,463 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:42,463 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:43,463 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:44,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:45,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:46,331 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:46,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:47,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:48,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:49,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:50,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:51,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:52,331 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:52,465 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:53,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:53,831 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:24:54,201 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:24:54,201 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:24:54,465 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:55,540 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:56,045 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:24:56,397 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:24:56,397 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_9f5ca32b8d4315bd9618.mp4 with policy now +2025-01-18 19:24:56,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:24:56,467 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:24:56,467 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:24:56,468 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:24:56,517 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:56,709 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/k33lvbg0-media/videos/eval/17_eval_17_9f5ca32b8d4315bd9618.mp4 +2025-01-18 19:24:57,046 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:24:57,046 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/17_eval_17_9f5ca32b8d4315bd9618.mp4 +2025-01-18 19:24:57,046 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:24:57,401 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/3vdpryqj-media/videos/eval/17_eval_17_9f5ca32b8d4315bd9618.mp4 +2025-01-18 19:24:57,478 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:57,621 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:58,046 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:24:58,512 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:59,537 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:00,509 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:01,548 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:02,048 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:02,543 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:03,319 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:03,523 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:04,539 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:05,526 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:06,049 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:06,550 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:07,552 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:08,363 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:08,551 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:09,224 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:25:09,225 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:25:09,589 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:10,051 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:25:10,363 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:25:10,363 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:25:10,363 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:25:10,615 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:11,051 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:25:11,584 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:12,052 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:12,595 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:13,604 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:14,005 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:14,052 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:14,575 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:15,593 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:16,053 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:16,601 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:17,581 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:18,615 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:19,355 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:19,619 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:20,600 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:21,615 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:22,610 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:23,608 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:23,831 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:25:24,216 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:25:24,216 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:25:24,361 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:24,606 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:25,606 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:26,057 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:26,607 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:27,606 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:28,607 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:29,361 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:29,607 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:30,607 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:31,607 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:32,607 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:33,607 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:34,361 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:34,608 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:35,608 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:36,608 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:37,608 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:38,608 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:39,216 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:25:39,216 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:25:39,361 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:39,608 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:40,608 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:41,609 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:42,609 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:43,609 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:44,361 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:44,609 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:45,609 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:46,609 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:47,631 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:48,628 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:48,824 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:25:48,824 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_fe8dc0d3e79f58b5302a.mp4 with policy now +2025-01-18 19:25:48,834 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:25:48,835 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:25:48,835 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:25:48,835 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:25:49,066 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:25:49,066 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/18_eval_18_fe8dc0d3e79f58b5302a.mp4 +2025-01-18 19:25:49,066 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:25:49,124 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/glcvjgia-media/videos/eval/18_eval_18_fe8dc0d3e79f58b5302a.mp4 +2025-01-18 19:25:49,364 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:49,449 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/8egwbfdo-media/videos/eval/18_eval_18_fe8dc0d3e79f58b5302a.mp4 +2025-01-18 19:25:49,653 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:50,067 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:50,689 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:51,758 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:52,067 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:52,685 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:53,656 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:53,832 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:25:54,068 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:54,255 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:25:54,255 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:25:54,412 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:54,662 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:55,713 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:56,717 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:57,709 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:58,070 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:58,070 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:25:58,421 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:25:58,422 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:25:58,422 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:25:58,694 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:59,070 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:25:59,701 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:00,071 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:26:00,404 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:00,695 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:01,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:02,072 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:26:02,689 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:03,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:04,072 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:26:04,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:05,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:06,388 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:06,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:07,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:08,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:09,221 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:26:09,221 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:26:09,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:10,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:11,388 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:11,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:12,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:13,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:14,691 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:15,691 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:16,388 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:16,691 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:17,691 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:18,691 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:19,691 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:20,691 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:21,388 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:21,692 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:22,693 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:23,692 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:23,832 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:26:24,221 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:26:24,221 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:26:24,692 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:25,692 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:26,388 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:26,693 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:27,693 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:28,693 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:29,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:30,708 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:30,805 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:26:30,805 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_98200c8b3f44031bae08.mp4 with policy now +2025-01-18 19:26:30,821 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:26:30,821 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:26:30,821 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:30,822 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:31,077 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/5xavx3hz-media/videos/eval/19_eval_19_98200c8b3f44031bae08.mp4 +2025-01-18 19:26:31,083 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:26:31,083 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/19_eval_19_98200c8b3f44031bae08.mp4 +2025-01-18 19:26:31,083 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:26:31,383 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/dao9cprm-media/videos/eval/19_eval_19_98200c8b3f44031bae08.mp4 +2025-01-18 19:26:31,706 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:31,920 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:32,083 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:26:32,781 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:33,766 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:34,793 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:35,757 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:36,085 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:26:36,793 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:37,697 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:37,783 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:38,781 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:39,087 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:26:39,260 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:26:39,260 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:26:39,471 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:26:39,471 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:39,472 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:39,774 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:40,087 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:26:40,087 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:26:40,087 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:26:40,812 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:41,341 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 19:26:41,341 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 19:26:41,508 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:26:41,774 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:41,802 INFO wandb-upload_0:2557979 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9w7qxo4k +2025-01-18 19:26:41,873 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 19:26:41,880 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:26:42,088 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:26:42,088 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:26:42,088 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/table/eval/success_rate_ci_20_e18b08b27993f2ae79a7.table.json +2025-01-18 19:26:42,089 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/table/eval/total_success_rate_ci_21_41785b785bae78c60d60.table.json +2025-01-18 19:26:42,089 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media +2025-01-18 19:26:42,089 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/table +2025-01-18 19:26:42,089 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/table/eval +2025-01-18 19:26:42,164 INFO SenderThread:2557979 [sender.py:send_request_log_artifact():1525] logged artifact run-u3lo36eq-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTg4ODA1MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzAyNTMyMQ==', 'latestArtifact': None}} +2025-01-18 19:26:42,164 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:26:42,164 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_e18b08b27993f2ae79a7.table.json with policy now +2025-01-18 19:26:42,165 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:26:42,165 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,165 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,165 DEBUG SenderThread:2557979 [sender.py:send():379] send: telemetry +2025-01-18 19:26:42,165 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,169 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,170 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,170 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,170 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,170 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,171 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,171 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,172 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,172 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,172 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,172 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,173 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,173 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,173 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,174 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,174 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,174 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,174 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,175 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,175 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,175 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,175 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 19:26:42,381 INFO wandb-upload_2:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/c9ow2w6m-media/table/eval/total_success_rate_ci_21_41785b785bae78c60d60.table.json +2025-01-18 19:26:42,402 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/0ozp5w34-media/table/eval/success_rate_ci_20_e18b08b27993f2ae79a7.table.json +2025-01-18 19:26:42,411 INFO wandb-upload_1:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/jaeudeso-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:26:42,611 INFO wandb-upload_2:2557979 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9ffc0x7r +2025-01-18 19:26:42,680 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/50fdirc7-media/table/eval/success_rate_ci_20_e18b08b27993f2ae79a7.table.json +2025-01-18 19:26:42,774 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:43,001 INFO SenderThread:2557979 [sender.py:send_request_log_artifact():1525] logged artifact run-u3lo36eq-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTg4ODA2Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzAyNTMyNg==', 'latestArtifact': None}} +2025-01-18 19:26:43,004 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:43,117 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:26:43,118 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_41785b785bae78c60d60.table.json with policy now +2025-01-18 19:26:43,118 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:26:43,118 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:43,118 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:43,119 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:26:43,370 INFO wandb-upload_1:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/3k67plp1-media/table/eval/total_success_rate_ci_21_41785b785bae78c60d60.table.json +2025-01-18 19:26:43,774 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:44,120 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/config.yaml +2025-01-18 19:26:44,120 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:26:44,120 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:26:44,775 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:45,775 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:46,775 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:47,775 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:48,390 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:48,775 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:49,775 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:50,775 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:51,775 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:52,775 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:53,776 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:53,832 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:26:53,834 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:54,242 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:26:54,242 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:26:54,776 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:55,776 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:56,776 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:57,776 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:58,117 DEBUG SenderThread:2557979 [sender.py:send():379] send: exit +2025-01-18 19:26:58,117 INFO SenderThread:2557979 [sender.py:send_exit():586] handling exit code: 0 +2025-01-18 19:26:58,118 INFO SenderThread:2557979 [sender.py:send_exit():588] handling runtime: 1084 +2025-01-18 19:26:58,118 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:58,118 INFO SenderThread:2557979 [sender.py:send_exit():594] send defer +2025-01-18 19:26:58,118 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:26:58,118 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-18 19:26:58,118 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:26:58,118 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-18 19:26:58,118 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 1 +2025-01-18 19:26:58,118 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:26:58,119 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-18 19:26:58,119 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:26:58,119 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-18 19:26:58,119 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 2 +2025-01-18 19:26:58,119 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:26:58,119 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-18 19:26:58,119 INFO HandlerThread:2557979 [system_monitor.py:finish():203] Stopping system monitor +2025-01-18 19:26:58,119 DEBUG SystemMonitor:2557979 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-18 19:26:58,119 DEBUG SystemMonitor:2557979 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-18 19:26:58,120 INFO HandlerThread:2557979 [interfaces.py:finish():200] Joined cpu monitor +2025-01-18 19:26:58,120 INFO HandlerThread:2557979 [interfaces.py:finish():200] Joined disk monitor +2025-01-18 19:26:58,127 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:26:58,170 INFO HandlerThread:2557979 [interfaces.py:finish():200] Joined gpu monitor +2025-01-18 19:26:58,170 INFO HandlerThread:2557979 [interfaces.py:finish():200] Joined memory monitor +2025-01-18 19:26:58,170 INFO HandlerThread:2557979 [interfaces.py:finish():200] Joined network monitor +2025-01-18 19:26:58,171 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:26:58,171 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-18 19:26:58,171 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 3 +2025-01-18 19:26:58,171 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:26:58,171 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:26:58,171 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-18 19:26:58,172 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:26:58,172 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-18 19:26:58,172 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 4 +2025-01-18 19:26:58,172 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:26:58,172 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-18 19:27:00,128 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:27:04,478 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:27:04,479 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:04,479 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-18 19:27:04,479 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 5 +2025-01-18 19:27:04,479 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:27:04,479 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:27:04,479 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: keepalive +2025-01-18 19:27:04,479 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:27:04,480 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:04,480 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-18 19:27:04,481 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:27:04,481 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:27:04,481 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:27:04,482 DEBUG SenderThread:2557979 [sender.py:send():379] send: summary +2025-01-18 19:27:04,486 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:27:04,486 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:04,486 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-18 19:27:04,486 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 6 +2025-01-18 19:27:04,486 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:04,486 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-18 19:27:04,486 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:04,486 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-18 19:27:04,486 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 7 +2025-01-18 19:27:04,486 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:27:04,486 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:04,487 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-18 19:27:04,487 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:04,487 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-18 19:27:05,119 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:27:05,130 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:27:05,746 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 8 +2025-01-18 19:27:05,746 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:27:05,746 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:05,746 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-18 19:27:05,746 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:05,746 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-18 19:27:05,746 INFO SenderThread:2557979 [job_builder.py:build():440] Attempting to build job artifact +2025-01-18 19:27:05,749 INFO SenderThread:2557979 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-18 19:27:05,751 INFO SenderThread:2557979 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-18 19:27:05,765 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 9 +2025-01-18 19:27:05,765 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:05,765 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-18 19:27:05,765 DEBUG SenderThread:2557979 [sender.py:send():379] send: artifact +2025-01-18 19:27:06,119 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:27:06,130 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:27:06,204 INFO wandb-upload_2:2557979 [upload_job.py:push():85] Skipped uploading /tmp/tmp1_2fxjxp/wandb-job.json +2025-01-18 19:27:06,204 INFO wandb-upload_0:2557979 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpavoy7qkt +2025-01-18 19:27:06,870 INFO SenderThread:2557979 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1MTg4ODMzOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1MTg1ODU1Ng==', 'versionIndex': 446}}} +2025-01-18 19:27:06,870 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:06,870 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-18 19:27:06,870 INFO SenderThread:2557979 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-18 19:27:07,131 INFO SenderThread:2557979 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files +2025-01-18 19:27:07,131 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/config.yaml config.yaml +2025-01-18 19:27:07,131 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:27:07,133 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json wandb-summary.json +2025-01-18 19:27:07,133 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/requirements.txt requirements.txt +2025-01-18 19:27:07,135 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-metadata.json wandb-metadata.json +2025-01-18 19:27:07,135 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log output.log +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/12_eval_12_9c363b6170975b374255.mp4 media/videos/eval/12_eval_12_9c363b6170975b374255.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/13_eval_13_c552251a3cda7642b754.mp4 media/videos/eval/13_eval_13_c552251a3cda7642b754.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/16_eval_16_a29e67cd0c3e166bf43e.mp4 media/videos/eval/16_eval_16_a29e67cd0c3e166bf43e.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/4_eval_4_edd1b447fab3432974b4.mp4 media/videos/eval/4_eval_4_edd1b447fab3432974b4.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/18_eval_18_fe8dc0d3e79f58b5302a.mp4 media/videos/eval/18_eval_18_fe8dc0d3e79f58b5302a.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/11_eval_11_5d5f6eccd1ac8a9641c2.mp4 media/videos/eval/11_eval_11_5d5f6eccd1ac8a9641c2.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/7_eval_7_d015da1851d414bb6628.mp4 media/videos/eval/7_eval_7_d015da1851d414bb6628.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/9_eval_9_d74f7b5c31b1fe3c2190.mp4 media/videos/eval/9_eval_9_d74f7b5c31b1fe3c2190.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/19_eval_19_98200c8b3f44031bae08.mp4 media/videos/eval/19_eval_19_98200c8b3f44031bae08.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/0_eval_0_f2d24d1f27d7648055dc.mp4 media/videos/eval/0_eval_0_f2d24d1f27d7648055dc.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/10_eval_10_741429cbea5a900591e5.mp4 media/videos/eval/10_eval_10_741429cbea5a900591e5.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/8_eval_8_9d16e439b84d1b213c76.mp4 media/videos/eval/8_eval_8_9d16e439b84d1b213c76.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/14_eval_14_58248ea2be38c14379cd.mp4 media/videos/eval/14_eval_14_58248ea2be38c14379cd.mp4 +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/15_eval_15_3efd5b9c2b7508acddc5.mp4 media/videos/eval/15_eval_15_3efd5b9c2b7508acddc5.mp4 +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/5_eval_5_c150bb925f103adcede9.mp4 media/videos/eval/5_eval_5_c150bb925f103adcede9.mp4 +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/1_eval_1_f85e4baf2746d94aa0a7.mp4 media/videos/eval/1_eval_1_f85e4baf2746d94aa0a7.mp4 +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/2_eval_2_340d87e376b6dfa888fb.mp4 media/videos/eval/2_eval_2_340d87e376b6dfa888fb.mp4 +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/17_eval_17_9f5ca32b8d4315bd9618.mp4 media/videos/eval/17_eval_17_9f5ca32b8d4315bd9618.mp4 +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/3_eval_3_a45dc41fdb9f639ccef4.mp4 media/videos/eval/3_eval_3_a45dc41fdb9f639ccef4.mp4 +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/6_eval_6_438f8d27cd8ba8c28d6a.mp4 media/videos/eval/6_eval_6_438f8d27cd8ba8c28d6a.mp4 +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/table/eval/total_success_rate_ci_21_41785b785bae78c60d60.table.json media/table/eval/total_success_rate_ci_21_41785b785bae78c60d60.table.json +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/table/eval/success_rate_ci_20_e18b08b27993f2ae79a7.table.json media/table/eval/success_rate_ci_20_e18b08b27993f2ae79a7.table.json +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 10 +2025-01-18 19:27:07,140 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:27:07,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:07,141 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-18 19:27:07,141 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:07,141 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-18 19:27:07,141 INFO SenderThread:2557979 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 19:27:07,308 INFO wandb-upload_2:2557979 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/requirements.txt +2025-01-18 19:27:07,327 INFO wandb-upload_3:2557979 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:27:07,348 INFO wandb-upload_1:2557979 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/config.yaml +2025-01-18 19:27:07,349 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:27:07,549 INFO Thread-11 (_thread_body):2557979 [sender.py:transition_state():614] send defer: 11 +2025-01-18 19:27:07,549 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:07,549 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-18 19:27:07,549 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:07,549 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-18 19:27:07,549 INFO SenderThread:2557979 [file_pusher.py:join():175] waiting for file pusher +2025-01-18 19:27:07,552 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 12 +2025-01-18 19:27:07,553 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:07,553 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-18 19:27:07,553 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:07,553 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-18 19:27:07,553 INFO SenderThread:2557979 [file_stream.py:finish():601] file stream finish called +2025-01-18 19:27:07,755 INFO SenderThread:2557979 [file_stream.py:finish():605] file stream finish is done +2025-01-18 19:27:07,755 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 13 +2025-01-18 19:27:07,755 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:07,755 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-18 19:27:07,755 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:07,755 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-18 19:27:07,755 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 14 +2025-01-18 19:27:07,755 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:07,755 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-18 19:27:07,756 DEBUG SenderThread:2557979 [sender.py:send():379] send: final +2025-01-18 19:27:07,756 DEBUG SenderThread:2557979 [sender.py:send():379] send: footer +2025-01-18 19:27:07,756 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:07,756 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-18 19:27:07,756 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:27:07,756 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: server_info +2025-01-18 19:27:07,756 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:27:07,757 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: server_info +2025-01-18 19:27:07,758 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:27:07,758 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: get_summary +2025-01-18 19:27:07,759 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-18 19:27:07,760 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:27:07,816 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:27:07,816 INFO MainThread:2557979 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-18 19:27:07,817 INFO MainThread:2557979 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-18 19:27:07,817 INFO MainThread:2557979 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-18 19:27:07,817 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: shutdown +2025-01-18 19:27:07,818 INFO HandlerThread:2557979 [handler.py:finish():882] shutting down handler +2025-01-18 19:27:08,758 INFO WriterThread:2557979 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/run-u3lo36eq.wandb +2025-01-18 19:27:08,816 INFO SenderThread:2557979 [sender.py:finish():1615] shutting down sender +2025-01-18 19:27:08,816 INFO SenderThread:2557979 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 19:27:08,816 INFO SenderThread:2557979 [file_pusher.py:join():175] waiting for file pusher diff --git a/u3lo36eq/wandb/debug.log b/u3lo36eq/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0ae07b57eb02879529824bfbcb4b675a932b0bc3 --- /dev/null +++ b/u3lo36eq/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_setup.py:_flush():76] Configure stats pid to 2557788 +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/settings +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/logs/debug.log +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/logs/debug-internal.log +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_init.py:init():569] calling init triggers +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_1', 'wandb_notes': '3465_1_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq'} +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_init.py:init():619] starting backend +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_init.py:init():623] setting up manager +2025-01-18 19:08:53,482 INFO MainThread:2557788 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-18 19:08:53,483 INFO MainThread:2557788 [wandb_init.py:init():631] backend started and connected +2025-01-18 19:08:53,495 INFO MainThread:2557788 [wandb_init.py:init():720] updated telemetry +2025-01-18 19:08:53,501 INFO MainThread:2557788 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-18 19:08:53,730 INFO MainThread:2557788 [wandb_run.py:_on_init():2435] communicating current version +2025-01-18 19:08:53,802 INFO MainThread:2557788 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-18 19:08:53,802 INFO MainThread:2557788 [wandb_init.py:init():804] starting run threads in backend +2025-01-18 19:08:54,076 INFO MainThread:2557788 [wandb_run.py:_console_start():2413] atexit reg +2025-01-18 19:08:54,076 INFO MainThread:2557788 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-18 19:08:54,077 INFO MainThread:2557788 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-18 19:08:54,077 INFO MainThread:2557788 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-18 19:08:54,078 INFO MainThread:2557788 [wandb_init.py:init():847] run started, returning control to user process +2025-01-18 19:08:54,079 INFO MainThread:2557788 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/tb, True +2025-01-18 19:08:58,120 INFO MainThread:2557788 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-18 19:27:08,818 WARNING MsgRouterThr:2557788 [router.py:message_loop():77] message_loop has been closed diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/config.yaml b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07d5b2db01c9eb558ece21fde558e4ac5fac7496 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/config.yaml @@ -0,0 +1,926 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '3465_1' +wandb_notes: + desc: null + value: 3465_1_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: null +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 20 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq +_wandb: + desc: null + value: + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737245333 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/table/eval/success_rate_ci_20_e18b08b27993f2ae79a7.table.json b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/table/eval/success_rate_ci_20_e18b08b27993f2ae79a7.table.json new file mode 100644 index 0000000000000000000000000000000000000000..08ccec415ae022d8c2386980b7ace1f9c135b4d3 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/table/eval/success_rate_ci_20_e18b08b27993f2ae79a7.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.25, 0.07984516032992633, 0.5252145528742905, 0.09151738890234472, 0.4560600966303012, 0.10851896189532234, 0.42931748928053204, 0.1642206088380032, 0.3712778687734557]]} \ No newline at end of file diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/table/eval/total_success_rate_ci_21_41785b785bae78c60d60.table.json b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/table/eval/total_success_rate_ci_21_41785b785bae78c60d60.table.json new file mode 100644 index 0000000000000000000000000000000000000000..4226749fdbc0d50f2a7a8ee329abd607b764b0f5 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/table/eval/total_success_rate_ci_21_41785b785bae78c60d60.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[5, 20, 0.25, 0.07259437450373979, 0.512881904837893, 0.10168606050828219, 0.45019891859096917, 0.10757853471140555, 0.45490397513814673, 0.128752332254988, 0.3927458524918488]]} \ No newline at end of file diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/0_eval_0_f2d24d1f27d7648055dc.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/0_eval_0_f2d24d1f27d7648055dc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a9fe5ec4e5a2da6c00833eda455c5e27e5e6d8cd --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/0_eval_0_f2d24d1f27d7648055dc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2d24d1f27d7648055dc3ada89105affb484179f0f09c9ba6f5d21d98c644d86 +size 1105784 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/10_eval_10_741429cbea5a900591e5.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/10_eval_10_741429cbea5a900591e5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..000b91b6798cfa604bda69384d68506c33599401 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/10_eval_10_741429cbea5a900591e5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:741429cbea5a900591e5d28a98fc2f0646f8799905bfaac36afaa369de494fb9 +size 1257587 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/11_eval_11_5d5f6eccd1ac8a9641c2.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/11_eval_11_5d5f6eccd1ac8a9641c2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0898f77539ba27a49c4331fd701682f755eeaf51 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/11_eval_11_5d5f6eccd1ac8a9641c2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d5f6eccd1ac8a9641c2ffa7c1563acbf5f91d531cbaf256c2447fb51ca55bb2 +size 1001294 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/12_eval_12_9c363b6170975b374255.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/12_eval_12_9c363b6170975b374255.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2446088d54a2872905494b95a05033a7b42850f2 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/12_eval_12_9c363b6170975b374255.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c363b6170975b374255612516dd2d909c80411d50b06f4ac9b21d84b2f795ec +size 1175492 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/13_eval_13_c552251a3cda7642b754.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/13_eval_13_c552251a3cda7642b754.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..51c04ec9faa21b2246e2119cac4f062cee203c0f --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/13_eval_13_c552251a3cda7642b754.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c552251a3cda7642b7546b8d1335e9ed0a120afbba48e3e84488db805833ef1c +size 1183827 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/14_eval_14_58248ea2be38c14379cd.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/14_eval_14_58248ea2be38c14379cd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e1bfe999aa15902b6f04ce3750c2d260f5c2c736 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/14_eval_14_58248ea2be38c14379cd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58248ea2be38c14379cda368756a3bb8731c389eeea3f5b49e12c09c7d7644bd +size 1455691 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/15_eval_15_3efd5b9c2b7508acddc5.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/15_eval_15_3efd5b9c2b7508acddc5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5c1173451b6072d3cc96280771b6f9af44c1026f --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/15_eval_15_3efd5b9c2b7508acddc5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3efd5b9c2b7508acddc5a6f0dcbc08a160cc8086197dc6d09621b7e5ff6e4af5 +size 1435091 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/16_eval_16_a29e67cd0c3e166bf43e.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/16_eval_16_a29e67cd0c3e166bf43e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8e22e4ded2cb92d774a34ea921b29968a4e55e77 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/16_eval_16_a29e67cd0c3e166bf43e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a29e67cd0c3e166bf43e22739fee1f09eba1a8a562eb67fbd669aa976b607bbe +size 1499720 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/17_eval_17_9f5ca32b8d4315bd9618.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/17_eval_17_9f5ca32b8d4315bd9618.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..592a5adfde09114777145bf90c0ce89315516f33 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/17_eval_17_9f5ca32b8d4315bd9618.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f5ca32b8d4315bd96189e8b5c0c9c42fa14207768d9deacd867cdbbcd94c15d +size 1682243 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/18_eval_18_fe8dc0d3e79f58b5302a.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/18_eval_18_fe8dc0d3e79f58b5302a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..afaba3f340116609ae8c043c0974c1e5f694449a --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/18_eval_18_fe8dc0d3e79f58b5302a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe8dc0d3e79f58b5302a3c322b3e34f0ea72380aee26862d037cf7f1ac15603d +size 1408117 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/19_eval_19_98200c8b3f44031bae08.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/19_eval_19_98200c8b3f44031bae08.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c1b5f2a2a8802919ddc939e09a3e7fab0fdcf1aa --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/19_eval_19_98200c8b3f44031bae08.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98200c8b3f44031bae085047c3976b35accd628515cc49e4bbd40a135554d5d4 +size 1176985 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/1_eval_1_f85e4baf2746d94aa0a7.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/1_eval_1_f85e4baf2746d94aa0a7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e63adc482c967815bcbfa06143a058a30b7a8b42 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/1_eval_1_f85e4baf2746d94aa0a7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f85e4baf2746d94aa0a7c8b7677ae095e9b50b375ccd51cd7d39fddf5749bbf5 +size 1069639 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/2_eval_2_340d87e376b6dfa888fb.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/2_eval_2_340d87e376b6dfa888fb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68abd28f925781e77089b969c4e5a2b64ae080b8 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/2_eval_2_340d87e376b6dfa888fb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:340d87e376b6dfa888fba755c925d901d2e44ae7f004557ae9589d089964e468 +size 1075858 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/3_eval_3_a45dc41fdb9f639ccef4.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/3_eval_3_a45dc41fdb9f639ccef4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ebc37efcb4d972972ee707a62596834f5aec6dac --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/3_eval_3_a45dc41fdb9f639ccef4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a45dc41fdb9f639ccef45d7a2e71ae53d23d26ec242ebc388da02b73366f5df3 +size 1155963 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/4_eval_4_edd1b447fab3432974b4.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/4_eval_4_edd1b447fab3432974b4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..31c3416dabb25562340049957e091f300c6930f2 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/4_eval_4_edd1b447fab3432974b4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edd1b447fab3432974b48be99a57b4dc183cd7a1aef9bccc0f20e680f96b6d2c +size 1241795 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/5_eval_5_c150bb925f103adcede9.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/5_eval_5_c150bb925f103adcede9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0a110c66eeaab03f08452095c3e9f8c42bc77f2d --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/5_eval_5_c150bb925f103adcede9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c150bb925f103adcede99be565e6a2271f8d73019cdeeff55855ffeacbe1952e +size 1202637 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/6_eval_6_438f8d27cd8ba8c28d6a.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/6_eval_6_438f8d27cd8ba8c28d6a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..372e0c2eb21c0c49c2ffb8fbeb56db6cf5bd41b3 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/6_eval_6_438f8d27cd8ba8c28d6a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:438f8d27cd8ba8c28d6a922dd0610eb5627087e78728af8b2dbb7d5e9e0775c2 +size 1250346 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/7_eval_7_d015da1851d414bb6628.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/7_eval_7_d015da1851d414bb6628.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d8daca8d8cbd4883f5ff509c172ffb3bacae752b --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/7_eval_7_d015da1851d414bb6628.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d015da1851d414bb66284e74bb9c0ca9bef27dd1ca7bd3169f2b998741a08d99 +size 1203795 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/8_eval_8_9d16e439b84d1b213c76.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/8_eval_8_9d16e439b84d1b213c76.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b3fcb545073977d445e718885dd5f27621585468 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/8_eval_8_9d16e439b84d1b213c76.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d16e439b84d1b213c76a32b6206b6b6d2b797bae4880b2182e5b6179572d8cf +size 1283069 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/9_eval_9_d74f7b5c31b1fe3c2190.mp4 b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/9_eval_9_d74f7b5c31b1fe3c2190.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..42e26c4ea8052a2a1443527a92d0dfb465d57802 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/9_eval_9_d74f7b5c31b1fe3c2190.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d74f7b5c31b1fe3c2190fc0a42914b3831fceb88147e73d6a09d33bd35a4f072 +size 1159824 diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..46703a45593f12ed04054d079cb4c43a12326815 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log @@ -0,0 +1,405 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737245338.121336]: resetting environment +[INFO] [1737245338.125340]: cleared current plan +[INFO] [1737245338.126170]: moving to home +[INFO] [1737245339.128980]: reached home +[INFO] [1737245339.130700]: reset action history +[INFO] [1737245341.839595]: environment reset +Starting episode 0 +[INFO] [1737245341.841647]: resetting environment +[INFO] [1737245341.846692]: cleared current plan +[INFO] [1737245341.847616]: moving to home +[INFO] [1737245342.849350]: reached home +[INFO] [1737245342.850843]: reset action history +[INFO] [1737245345.558954]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1737245359.123655]: resetting environment +[INFO] [1737245359.130256]: cleared current plan +[INFO] [1737245359.132203]: moving to home +[INFO] [1737245360.135808]: reached home +[INFO] [1737245360.137641]: reset action history +[INFO] [1737245362.895878]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/episode_rosbags/episode_0_2025-01-18-19-09-22.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2557788.log +started roslaunch server http://158.130.50.37:37271/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2558058] +process[print_text-2]: started with pid [2558081] +process[rosbag_record-3]: started with pid [2558083] +started bagging! +For topic gripper_width: timestamp difference is -127818833 for nearest: 1737245368802296390 - target: 1737245368930115223 at idx 80 +preempting episode +[INFO] [1737245387.507838]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1737245388.315463]: Stored episode 1. +[INFO] [1737245388.316273]: Saving video... +[INFO] [1737245388.531201]: Video saved! +Starting episode 1 +[INFO] [1737245388.543947]: resetting environment +[INFO] [1737245388.548257]: cleared current plan +[INFO] [1737245388.549067]: moving to home +[INFO] [1737245391.951554]: reached home +[INFO] [1737245391.952009]: reset action history +[INFO] [1737245394.661146]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737245406.350239]: resetting environment +[INFO] [1737245406.353593]: cleared current plan +[INFO] [1737245406.353754]: moving to home +[INFO] [1737245407.355217]: reached home +[INFO] [1737245407.355886]: reset action history +[INFO] [1737245410.062184]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/episode_rosbags/episode_1_2025-01-18-19-10-10.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2557788.log +started roslaunch server http://158.130.50.37:42529/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2558189] +process[print_text-5]: started with pid [2558190] +process[rosbag_record-6]: started with pid [2558191] +started bagging! +For topic gripper_width: timestamp difference is -115316008 for nearest: 1737245423002338554 - target: 1737245423117654562 at idx 173 +For topic gripper_width: timestamp difference is -147120931 for nearest: 1737245427802290223 - target: 1737245427949411154 at idx 228 +preempting episode +[INFO] [1737245433.594387]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1737245434.366619]: Stored episode 2. +[INFO] [1737245434.367236]: Saving video... +[INFO] [1737245434.925439]: Video saved! +Starting episode 2 +[INFO] [1737245434.953550]: resetting environment +[INFO] [1737245434.965045]: cleared current plan +[INFO] [1737245434.966183]: moving to home +[WARN] [1737245434.751975]: Plan exhausted +[WARN] [1737245434.794149]: Plan exhausted +[WARN] [1737245434.831968]: Plan exhausted +[WARN] [1737245434.873847]: Plan exhausted +[WARN] [1737245434.911948]: Plan exhausted +[WARN] [1737245434.953104]: Plan exhausted +[INFO] [1737245438.371905]: reached home +[INFO] [1737245438.373598]: reset action history +[INFO] [1737245441.089878]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737245444.550235]: resetting environment +[INFO] [1737245444.558645]: cleared current plan +[INFO] [1737245444.558878]: moving to home +[INFO] [1737245445.560642]: reached home +[INFO] [1737245445.561495]: reset action history +[INFO] [1737245448.320840]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/episode_rosbags/episode_2_2025-01-18-19-10-48.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2557788.log +started roslaunch server http://158.130.50.37:32851/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2558327] +process[print_text-8]: started with pid [2558328] +process[rosbag_record-9]: started with pid [2558329] +started bagging! +For topic gripper_width: timestamp difference is 103180238 for nearest: 1737245449068962642 - target: 1737245448965782404 at idx 12 +For topic gripper_width: timestamp difference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +For topic gripper_width: timestamp difference is 104960449 for nearest: 1737245451268963106 - target: 1737245451164002657 at idx 44 +For topic gripper_width: timestamp difference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +For topic gripper_width: timestamp difference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +[WARN] [1737245473.790959]: Plan exhausted +[WARN] [1737245473.831776]: Plan exhausted +[WARN] [1737245473.873862]: Plan exhausted +For topic gripper_width: timestamp difference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +Comuting lower confidence boundamp difference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +[WARN] [1737245518.911138]: Plan exhausted +[WARN] [1737245518.952381]: Plan exhausted +[WARN] [1737245518.992888]: Plan exhausted +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +[WARN] [1737245614.718706]: Plan exhausted +[WARN] [1737245614.751289]: Plan exhausted +[WARN] [1737245614.792665]: Plan exhausted +[WARN] [1737245614.834235]: Plan exhausted +[WARN] [1737245614.871277]: Plan exhausted +[WARN] [1737245614.912362]: Plan exhausted +[WARN] [1737245614.955818]: Plan exhausted +[WARN] [1737245614.990733]: Plan exhausted +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +[WARN] [1737245702.430743]: Plan exhausted +[WARN] [1737245702.471697]: Plan exhausted +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +[WARN] [1737245758.390775]: Plan exhausted +[WARN] [1737245758.435363]: Plan exhausted +[WARN] [1737245758.471333]: Plan exhausted +[WARN] [1737245758.513664]: Plan exhausted +[WARN] [1737245758.550736]: Plan exhausted +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +[WARN] [1737245758.598776]: Plan exhausted +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +[WARN] [1737245803.871885]: Plan exhausted +[WARN] [1737245803.910818]: Plan exhausted +[WARN] [1737245803.952037]: Plan exhausted +[WARN] [1737245803.992718]: Plan exhausted +[WARN] [1737245804.034324]: Plan exhausted +[WARN] [1737245804.070726]: Plan exhausted +[WARN] [1737245804.120707]: Plan exhausted +[WARN] [1737245804.155836]: Plan exhausted +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +[WARN] [1737245866.512120]: Plan exhausted +[WARN] [1737245866.556197]: Plan exhausted +[WARN] [1737245866.610652]: Plan exhausted +[WARN] [1737245866.630730]: Plan exhausted +[WARN] [1737245866.670730]: Plan exhausted +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +[WARN] [1737245921.573900]: Plan exhausted +[WARN] [1737245921.616490]: Plan exhausted +[WARN] [1737245921.643159]: Plan exhausted +[WARN] [1737245921.674249]: Plan exhausted +[WARN] [1737245921.713755]: Plan exhausted +[WARN] [1737245921.758265]: Plan exhausted +[WARN] [1737245921.792724]: Plan exhausted +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +[WARN] [1737245969.471405]: Plan exhausted +[WARN] [1737245969.521014]: Plan exhausted +[WARN] [1737245969.575260]: Plan exhausted +[WARN] [1737245969.601909]: Plan exhausted +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +[WARN] [1737246012.158854]: Plan exhausted +[WARN] [1737246012.201532]: Plan exhausted +[WARN] [1737246012.257590]: Plan exhausted +[WARN] [1737246012.275823]: Plan exhausted +[WARN] [1737246012.319793]: Plan exhausted +[WARN] [1737246012.357682]: Plan exhausted +[WARN] [1737246012.393860]: Plan exhausted +[WARN] [1737246012.444018]: Plan exhausted +[WARN] [1737246012.477773]: Plan exhausted +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +[WARN] [1737246127.164751]: Plan exhausted +[WARN] [1737246127.190759]: Plan exhausted +[WARN] [1737246127.234626]: Plan exhausted +[WARN] [1737246127.271976]: Plan exhausted +[WARN] [1737246127.335114]: Plan exhausted +[WARN] [1737246127.401986]: Plan exhausted +[WARN] [1737246127.412427]: Plan exhausted +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +[WARN] [1737246179.072346]: Plan exhausted +[WARN] [1737246179.119307]: Plan exhausted +[WARN] [1737246179.150727]: Plan exhausted +[WARN] [1737246179.195647]: Plan exhausted +[WARN] [1737246179.253535]: Plan exhausted +[WARN] [1737246179.272827]: Plan exhausted +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +[WARN] [1737246237.492347]: Plan exhausted +[WARN] [1737246237.511596]: Plan exhausted +[WARN] [1737246237.556716]: Plan exhausted +[WARN] [1737246237.596418]: Plan exhausted +[WARN] [1737246237.633932]: Plan exhausted +[WARN] [1737246237.670724]: Plan exhausted +[WARN] [1737246237.713394]: Plan exhausted +[WARN] [1737246237.760506]: Plan exhausted +[WARN] [1737246237.796051]: Plan exhausted +[WARN] [1737246237.833548]: Plan exhausted +[WARN] [1737246237.875767]: Plan exhausted +[WARN] [1737246237.910922]: Plan exhausted +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +[WARN] [1737246295.991841]: Plan exhausted +[WARN] [1737246296.035192]: Plan exhausted +[WARN] [1737246296.071532]: Plan exhausted +[WARN] [1737246296.116923]: Plan exhausted +[WARN] [1737246296.152097]: Plan exhausted +[WARN] [1737246296.191623]: Plan exhausted +[WARN] [1737246296.236725]: Plan exhausted +[WARN] [1737246296.282952]: Plan exhausted +[WARN] [1737246296.342860]: Plan exhausted +[WARN] [1737246296.360295]: Plan exhausted +[WARN] [1737246296.393220]: Plan exhausted +[WARN] [1737246296.433644]: Plan exhausted +[WARN] [1737246296.495734]: Plan exhausted +[WARN] [1737246296.546751]: Plan exhausted +[WARN] [1737246296.561498]: Plan exhausted +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +[WARN] [1737246348.522252]: Plan exhausted +[WARN] [1737246348.552251]: Plan exhausted +[WARN] [1737246348.590847]: Plan exhausted +[WARN] [1737246348.633107]: Plan exhausted +[WARN] [1737246348.684976]: Plan exhausted +[WARN] [1737246348.733415]: Plan exhausted +[WARN] [1737246348.774295]: Plan exhausted +[WARN] [1737246348.794815]: Plan exhausted +[WARN] [1737246348.830725]: Plan exhausted +[WARN] [1737246348.913499]: Plan exhausted +[WARN] [1737246348.940298]: Plan exhausted +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +[WARN] [1737246390.751545]: Plan exhausted +[WARN] [1737246390.797657]: Plan exhausted +[WARN] [1737246390.863433]: Plan exhausted +[WARN] [1737246390.888566]: Plan exhausted +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 +ROS_MASTER_URI=http://localhost:11311fference is 103856320 for nearest: 1737245449869001145 - target: 1737245449765144825 at idx 24 \ No newline at end of file diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/requirements.txt b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-metadata.json b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4c67ecd023cfb45cbee6944bc434b59f28614f4a --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-19T00:08:53.827800", + "startedAt": "2025-01-19T00:08:53.469225", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "ec42957dc10d08dbd8b134558fcd8f9e470ccc31" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3383.22575, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4543.658, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 524.6030311584473 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ec6275408badc5922ea3af14120a64faa15c01f0 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "f2d24d1f27d7648055dc3ada89105affb484179f0f09c9ba6f5d21d98c644d86", "size": 1105784, "path": "media/videos/eval/0_eval_0_f2d24d1f27d7648055dc.mp4"}, "global_step": 2571, "_timestamp": 1737246401.8006763, "_runtime": 1068.3167703151703, "_step": 41, "eval/1_eval": {"_type": "video-file", "sha256": "f85e4baf2746d94aa0a7c8b7677ae095e9b50b375ccd51cd7d39fddf5749bbf5", "size": 1069639, "path": "media/videos/eval/1_eval_1_f85e4baf2746d94aa0a7.mp4"}, "eval/num_success": 5.0, "episode": 20.0, "eval/success_rate": 0.25, "eval/success_rate_lb_0.01": 0.07984516024589539, "eval/success_rate_ub_0.01": 0.5252145528793335, "eval/success_rate_lb_0.025": 0.0915173888206482, "eval/success_rate_ub_0.025": 0.45606011152267456, "eval/success_rate_lb_0.05": 0.10851896554231644, "eval/success_rate_ub_0.05": 0.42931750416755676, "eval/success_rate_lb_0.1": 0.16422061622142792, "eval/success_rate_ub_0.1": 0.3712778687477112, "eval/2_eval": {"_type": "video-file", "sha256": "340d87e376b6dfa888fba755c925d901d2e44ae7f004557ae9589d089964e468", "size": 1075858, "path": "media/videos/eval/2_eval_2_340d87e376b6dfa888fb.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "a45dc41fdb9f639ccef45d7a2e71ae53d23d26ec242ebc388da02b73366f5df3", "size": 1155963, "path": "media/videos/eval/3_eval_3_a45dc41fdb9f639ccef4.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "edd1b447fab3432974b48be99a57b4dc183cd7a1aef9bccc0f20e680f96b6d2c", "size": 1241795, "path": "media/videos/eval/4_eval_4_edd1b447fab3432974b4.mp4"}, "eval/5_eval": {"_type": "video-file", "sha256": "c150bb925f103adcede99be565e6a2271f8d73019cdeeff55855ffeacbe1952e", "size": 1202637, "path": "media/videos/eval/5_eval_5_c150bb925f103adcede9.mp4"}, "eval/6_eval": {"_type": "video-file", "sha256": "438f8d27cd8ba8c28d6a922dd0610eb5627087e78728af8b2dbb7d5e9e0775c2", "size": 1250346, "path": "media/videos/eval/6_eval_6_438f8d27cd8ba8c28d6a.mp4"}, "eval/7_eval": {"_type": "video-file", "sha256": "d015da1851d414bb66284e74bb9c0ca9bef27dd1ca7bd3169f2b998741a08d99", "size": 1203795, "path": "media/videos/eval/7_eval_7_d015da1851d414bb6628.mp4"}, "eval/8_eval": {"_type": "video-file", "sha256": "9d16e439b84d1b213c76a32b6206b6b6d2b797bae4880b2182e5b6179572d8cf", "size": 1283069, "path": "media/videos/eval/8_eval_8_9d16e439b84d1b213c76.mp4"}, "eval/9_eval": {"_type": "video-file", "sha256": "d74f7b5c31b1fe3c2190fc0a42914b3831fceb88147e73d6a09d33bd35a4f072", "size": 1159824, "path": "media/videos/eval/9_eval_9_d74f7b5c31b1fe3c2190.mp4"}, "eval/10_eval": {"_type": "video-file", "sha256": "741429cbea5a900591e5d28a98fc2f0646f8799905bfaac36afaa369de494fb9", "size": 1257587, "path": "media/videos/eval/10_eval_10_741429cbea5a900591e5.mp4"}, "eval/11_eval": {"_type": "video-file", "sha256": "5d5f6eccd1ac8a9641c2ffa7c1563acbf5f91d531cbaf256c2447fb51ca55bb2", "size": 1001294, "path": "media/videos/eval/11_eval_11_5d5f6eccd1ac8a9641c2.mp4"}, "eval/12_eval": {"_type": "video-file", "sha256": "9c363b6170975b374255612516dd2d909c80411d50b06f4ac9b21d84b2f795ec", "size": 1175492, "path": "media/videos/eval/12_eval_12_9c363b6170975b374255.mp4"}, "eval/13_eval": {"_type": "video-file", "sha256": "c552251a3cda7642b7546b8d1335e9ed0a120afbba48e3e84488db805833ef1c", "size": 1183827, "path": "media/videos/eval/13_eval_13_c552251a3cda7642b754.mp4"}, "eval/14_eval": {"_type": "video-file", "sha256": "58248ea2be38c14379cda368756a3bb8731c389eeea3f5b49e12c09c7d7644bd", "size": 1455691, "path": "media/videos/eval/14_eval_14_58248ea2be38c14379cd.mp4"}, "eval/15_eval": {"_type": "video-file", "sha256": "3efd5b9c2b7508acddc5a6f0dcbc08a160cc8086197dc6d09621b7e5ff6e4af5", "size": 1435091, "path": "media/videos/eval/15_eval_15_3efd5b9c2b7508acddc5.mp4"}, "eval/16_eval": {"_type": "video-file", "sha256": "a29e67cd0c3e166bf43e22739fee1f09eba1a8a562eb67fbd669aa976b607bbe", "size": 1499720, "path": "media/videos/eval/16_eval_16_a29e67cd0c3e166bf43e.mp4"}, "eval/17_eval": {"_type": "video-file", "sha256": "9f5ca32b8d4315bd96189e8b5c0c9c42fa14207768d9deacd867cdbbcd94c15d", "size": 1682243, "path": "media/videos/eval/17_eval_17_9f5ca32b8d4315bd9618.mp4"}, "eval/18_eval": {"_type": "video-file", "sha256": "fe8dc0d3e79f58b5302a3c322b3e34f0ea72380aee26862d037cf7f1ac15603d", "size": 1408117, "path": "media/videos/eval/18_eval_18_fe8dc0d3e79f58b5302a.mp4"}, "eval/19_eval": {"_type": "video-file", "sha256": "98200c8b3f44031bae085047c3976b35accd628515cc49e4bbd40a135554d5d4", "size": 1176985, "path": "media/videos/eval/19_eval_19_98200c8b3f44031bae08.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "e18b08b27993f2ae79a7a0394da39dc2a9c845471b3e00c68a2cd740479c5690", "size": 403, "artifact_path": "wandb-client-artifact://nw5uir7ug2mcv491dxf0gxvom8o110e2ghptcsory59x4ew6dvq4ki463r9390klp1jehd1ajff8v7yjrellsw5wu1m5jyn2egnonx4p0ij2x73nlrjlu2jktvsmh31a/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://7ysm6gw763txfrtxqx6kgfqt6pbswecl921r02hjzcrgfxwnjn7ou08evns2bi87d39jbidroxjbeqld10cpww84pgifz5tacd2kld3f3be6coyi03zqriwxoplwdp89:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_20_e18b08b27993f2ae79a7.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 5, "total_num_episodes": 20, "total_success_rate": 0.25, "total_success_rate_lb_0.01": 0.07259437450373979, "total_success_rate_ub_0.01": 0.512881904837893, "total_success_rate_lb_0.025": 0.10168606050828219, "total_success_rate_ub_0.025": 0.45019891859096917, "total_success_rate_lb_0.05": 0.10757853471140555, "total_success_rate_ub_0.05": 0.45490397513814673, "total_success_rate_lb_0.1": 0.128752332254988, "total_success_rate_ub_0.1": 0.3927458524918488, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "41785b785bae78c60d6010361e82ad2c3df00d5469a5c37de016049a0a779e79", "size": 508, "artifact_path": "wandb-client-artifact://rcqoh8eo25tlp0vvap6xhabl70uw455ol4pzy227ob1wnb2lkepvljx31dcrfetpyucr2wy51mkrbz8p4bvy99h0w2mppjf2wvepuliq588nmglfsd2h89c25xemfpmv/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://p8j8f3aeauypapihyrd0esg347v6hwbv13zjb2dnbvoo6mbup7ah5juk6xvku95tfre1c8oa9tay8434zzp1lctx9ra7nziani3dyj301w8pedmydf0zp5e57go0c0e5:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_21_41785b785bae78c60d60.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 1084}, "eval/total_success_rate": 0.25, "eval/total_success_rate_lb_0.01": 0.07259437441825867, "eval/total_success_rate_ub_0.01": 0.512881875038147, "eval/total_success_rate_lb_0.025": 0.10168606042861938, "eval/total_success_rate_ub_0.025": 0.4501989185810089, "eval/total_success_rate_lb_0.05": 0.10757853835821152, "eval/total_success_rate_ub_0.05": 0.4549039900302887, "eval/total_success_rate_lb_0.1": 0.12875233590602875, "eval/total_success_rate_ub_0.1": 0.39274585247039795} \ No newline at end of file diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/logs/debug-internal.log b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..983a2d3bab131751666b486e9c8f490ca1b52573 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/logs/debug-internal.log @@ -0,0 +1,2258 @@ +2025-01-18 19:08:53,484 INFO StreamThr :2557979 [internal.py:wandb_internal():85] W&B internal server running at pid: 2557979, started at: 2025-01-18 19:08:53.483861 +2025-01-18 19:08:53,485 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status +2025-01-18 19:08:53,496 INFO WriterThread:2557979 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/run-u3lo36eq.wandb +2025-01-18 19:08:53,496 DEBUG SenderThread:2557979 [sender.py:send():379] send: header +2025-01-18 19:08:53,503 DEBUG SenderThread:2557979 [sender.py:send():379] send: run +2025-01-18 19:08:53,707 INFO SenderThread:2557979 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files +2025-01-18 19:08:53,707 INFO SenderThread:2557979 [sender.py:_start_run_threads():1188] run started: u3lo36eq with start time 1737245333.483906 +2025-01-18 19:08:53,731 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: check_version +2025-01-18 19:08:53,731 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: check_version +2025-01-18 19:08:53,808 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: run_start +2025-01-18 19:08:53,813 DEBUG HandlerThread:2557979 [system_info.py:__init__():26] System info init +2025-01-18 19:08:53,813 DEBUG HandlerThread:2557979 [system_info.py:__init__():41] System info init done +2025-01-18 19:08:53,813 INFO HandlerThread:2557979 [system_monitor.py:start():194] Starting system monitor +2025-01-18 19:08:53,813 INFO SystemMonitor:2557979 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-18 19:08:53,814 INFO HandlerThread:2557979 [system_monitor.py:probe():214] Collecting system info +2025-01-18 19:08:53,814 INFO SystemMonitor:2557979 [interfaces.py:start():188] Started cpu monitoring +2025-01-18 19:08:53,815 INFO SystemMonitor:2557979 [interfaces.py:start():188] Started disk monitoring +2025-01-18 19:08:53,815 INFO SystemMonitor:2557979 [interfaces.py:start():188] Started gpu monitoring +2025-01-18 19:08:53,816 INFO SystemMonitor:2557979 [interfaces.py:start():188] Started memory monitoring +2025-01-18 19:08:53,816 INFO SystemMonitor:2557979 [interfaces.py:start():188] Started network monitoring +2025-01-18 19:08:53,827 DEBUG HandlerThread:2557979 [system_info.py:probe():152] Probing system +2025-01-18 19:08:53,829 DEBUG HandlerThread:2557979 [system_info.py:_probe_git():137] Probing git +2025-01-18 19:08:53,833 DEBUG HandlerThread:2557979 [system_info.py:_probe_git():145] Probing git done +2025-01-18 19:08:53,833 DEBUG HandlerThread:2557979 [system_info.py:probe():200] Probing system done +2025-01-18 19:08:53,833 DEBUG HandlerThread:2557979 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-19T00:08:53.827800', 'startedAt': '2025-01-19T00:08:53.469225', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3383.22575, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4543.658, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 524.6030311584473}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-18 19:08:53,833 INFO HandlerThread:2557979 [system_monitor.py:probe():224] Finished collecting system info +2025-01-18 19:08:53,834 INFO HandlerThread:2557979 [system_monitor.py:probe():227] Publishing system info +2025-01-18 19:08:53,834 INFO HandlerThread:2557979 [system_monitor.py:probe():229] Finished publishing system info +2025-01-18 19:08:53,837 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:08:53,837 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-18 19:08:54,075 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: python_packages +2025-01-18 19:08:54,076 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: python_packages +2025-01-18 19:08:54,087 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/vbgd9pni-wandb-metadata.json +2025-01-18 19:08:54,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:08:54,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:08:54,117 INFO HandlerThread:2557979 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/tb" + save: true +} +control { + relay_id: "60f290ddaf4c43f8ad9368cf744d939e" +} +_info { + stream_id: "u3lo36eq" +} + +2025-01-18 19:08:54,117 DEBUG HandlerThread:2557979 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-18 19:08:54,118 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:08:54,228 DEBUG SenderThread:2557979 [sender.py:send():379] send: telemetry +2025-01-18 19:08:54,228 DEBUG SenderThread:2557979 [sender.py:send():379] send: tbrecord +2025-01-18 19:08:54,229 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:08:54,229 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 with policy live +2025-01-18 19:08:54,406 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/mp593bay-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:08:54,710 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-metadata.json +2025-01-18 19:08:54,710 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:08:54,710 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/requirements.txt +2025-01-18 19:08:54,710 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:08:55,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:08:56,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:08:56,708 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:08:57,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:08:58,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:08:58,121 DEBUG SenderThread:2557979 [sender.py:send():379] send: config +2025-01-18 19:08:59,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:08:59,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:00,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:00,709 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:01,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:02,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:03,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:04,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:04,710 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:05,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:05,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:06,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:06,711 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:07,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:08,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:09,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:09:09,076 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:09:09,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:10,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:10,152 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:11,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:12,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:13,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:14,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:15,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:16,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:16,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:17,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:18,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:19,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:20,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:20,714 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:21,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:21,177 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:22,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:22,714 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:23,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:24,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:09:24,076 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:09:24,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:24,715 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:25,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:26,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:26,191 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:26,715 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/config.yaml +2025-01-18 19:09:27,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:28,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:29,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:30,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:30,716 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:31,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:32,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:32,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:33,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:34,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:35,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:36,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:37,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:37,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:38,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:39,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:09:39,076 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:09:39,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:40,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:41,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:42,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:42,144 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:43,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:44,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:45,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:46,083 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:47,083 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:47,496 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:48,083 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:48,542 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:09:48,542 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:09:48,542 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_f2d24d1f27d7648055dc.mp4 with policy now +2025-01-18 19:09:48,543 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:09:48,543 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:09:48,543 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:09:48,720 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:48,720 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/0_eval_0_f2d24d1f27d7648055dc.mp4 +2025-01-18 19:09:48,720 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:09:48,721 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media +2025-01-18 19:09:48,721 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:09:48,721 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos +2025-01-18 19:09:48,815 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/ruzqdu7e-media/videos/eval/0_eval_0_f2d24d1f27d7648055dc.mp4 +2025-01-18 19:09:49,083 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:49,100 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/9gurev3a-media/videos/eval/0_eval_0_f2d24d1f27d7648055dc.mp4 +2025-01-18 19:09:50,083 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:50,721 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:51,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:52,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:52,993 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:53,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:53,816 DEBUG SystemMonitor:2557979 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-18 19:09:53,817 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:09:54,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:09:54,077 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:09:54,125 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:54,722 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:55,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:56,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:56,723 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:09:57,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:58,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:09:58,706 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:09:59,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:00,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:01,085 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:02,085 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:03,085 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:04,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:04,085 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:05,085 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:06,085 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:06,726 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:10:06,972 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/8oy5cb5r-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:10:07,085 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:08,085 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:08,727 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:09,076 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:10:09,077 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:10:09,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:09,223 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:10,105 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:11,086 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:12,086 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:12,728 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:13,086 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:14,086 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:15,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:15,086 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:16,087 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:17,087 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:18,087 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:19,087 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:20,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:20,087 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:21,087 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:22,087 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:23,087 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:23,826 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:10:24,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:10:24,077 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:10:24,087 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:24,731 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:25,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:25,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:26,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:27,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:28,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:29,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:30,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:30,733 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:31,079 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:31,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:32,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:33,089 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:34,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:34,734 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:34,951 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:10:34,951 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:10:34,951 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f85e4baf2746d94aa0a7.mp4 with policy now +2025-01-18 19:10:34,952 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:10:34,952 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:10:34,952 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:10:35,092 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:35,215 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/511xv1rb-media/videos/eval/1_eval_1_f85e4baf2746d94aa0a7.mp4 +2025-01-18 19:10:35,735 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:10:35,735 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/1_eval_1_f85e4baf2746d94aa0a7.mp4 +2025-01-18 19:10:35,735 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:10:36,051 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/cgbbm1q3-media/videos/eval/1_eval_1_f85e4baf2746d94aa0a7.mp4 +2025-01-18 19:10:36,093 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:36,735 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:36,968 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:37,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:37,736 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:38,093 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:39,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:10:39,080 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:10:39,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:40,093 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:40,736 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:41,133 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:42,093 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:42,134 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:42,737 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:43,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:44,093 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:44,738 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:10:45,093 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:45,518 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:10:45,518 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:10:45,519 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:10:45,738 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:10:46,093 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:46,738 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:47,093 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:47,601 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:48,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:49,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:50,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:50,739 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:51,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:52,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:52,740 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:10:53,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:53,317 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:53,818 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:10:54,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:10:54,078 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:10:54,121 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:55,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:56,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:57,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:58,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:10:59,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:10:59,094 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:00,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:01,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:02,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:03,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:04,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:05,081 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:05,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:06,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:07,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:08,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:09,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:11:09,077 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:11:09,121 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:10,096 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:10,200 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:11,096 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:12,096 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:13,098 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:13,866 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:11:13,866 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_340d87e376b6dfa888fb.mp4 with policy now +2025-01-18 19:11:13,869 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:11:13,869 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:11:13,870 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:11:13,871 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:11:14,098 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:14,126 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/vmi8qezn-media/videos/eval/2_eval_2_340d87e376b6dfa888fb.mp4 +2025-01-18 19:11:14,747 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:11:14,747 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:11:14,747 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/2_eval_2_340d87e376b6dfa888fb.mp4 +2025-01-18 19:11:14,748 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:11:15,063 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/cnt70odx-media/videos/eval/2_eval_2_340d87e376b6dfa888fb.mp4 +2025-01-18 19:11:15,103 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:15,747 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:11:16,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:16,109 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:17,103 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:18,106 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:18,748 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:11:19,110 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:20,110 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:21,108 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:21,209 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:22,114 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:22,750 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:11:23,114 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:23,818 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:11:24,078 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:11:24,078 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:11:24,125 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:25,112 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:26,111 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:26,215 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:26,751 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:11:26,998 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/k1aae7z8-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:11:27,113 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:27,351 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:11:27,351 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:11:27,352 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:11:27,751 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:11:28,116 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:28,751 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:11:29,116 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:30,118 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:31,116 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:31,876 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:32,115 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:32,753 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:11:33,115 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:34,116 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:34,753 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:11:35,116 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:36,116 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:37,083 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:37,116 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:38,116 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:39,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:11:39,078 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:11:39,121 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:40,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:41,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:42,116 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:42,207 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:43,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:44,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:45,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:46,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:47,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:48,083 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:48,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:49,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:50,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:51,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:52,118 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:53,084 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:53,118 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:53,819 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:11:54,077 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:11:54,078 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:11:54,121 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:55,118 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:56,118 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:57,118 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:58,120 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:58,378 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:11:58,760 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:11:58,984 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:11:58,984 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a45dc41fdb9f639ccef4.mp4 with policy now +2025-01-18 19:11:58,984 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:11:58,985 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:11:58,985 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:11:58,986 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:11:59,120 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:11:59,265 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/vzvvmmix-media/videos/eval/3_eval_3_a45dc41fdb9f639ccef4.mp4 +2025-01-18 19:11:59,761 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:11:59,761 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/3_eval_3_a45dc41fdb9f639ccef4.mp4 +2025-01-18 19:11:59,761 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:12:00,107 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/63kl3bky-media/videos/eval/3_eval_3_a45dc41fdb9f639ccef4.mp4 +2025-01-18 19:12:00,125 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:00,761 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:01,125 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:01,761 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:02,137 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:03,131 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:03,565 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:04,129 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:04,762 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:05,136 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:06,129 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:06,763 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:07,130 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:08,131 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:09,083 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:12:09,083 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:12:09,131 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:09,221 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:10,134 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:11,133 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:12,133 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:13,139 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:14,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:15,090 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:15,138 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:16,139 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:17,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:18,138 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:18,414 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:12:18,414 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:12:18,414 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:12:18,766 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:12:18,766 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:18,766 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:12:19,137 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:20,091 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:20,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:20,767 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:21,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:22,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:22,767 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:23,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:23,819 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:12:24,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:12:24,080 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:12:24,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:24,768 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:25,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:25,152 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:26,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:27,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:28,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:28,770 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:29,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:30,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:31,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:31,141 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:32,141 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:33,141 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:34,141 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:35,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:36,088 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:36,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:37,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:38,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:39,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:12:39,081 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:12:39,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:40,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:41,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:41,180 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:42,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:43,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:44,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:45,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:46,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:47,089 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:47,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:48,144 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:49,144 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:50,144 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:51,149 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:51,625 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:12:51,625 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:12:51,625 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_edd1b447fab3432974b4.mp4 with policy now +2025-01-18 19:12:51,634 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:12:51,635 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:12:51,636 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:12:51,777 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:12:51,777 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/4_eval_4_edd1b447fab3432974b4.mp4 +2025-01-18 19:12:51,778 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:12:51,896 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/mlhtw6xu-media/videos/eval/4_eval_4_edd1b447fab3432974b4.mp4 +2025-01-18 19:12:52,162 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:52,309 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/z7dphjvn-media/videos/eval/4_eval_4_edd1b447fab3432974b4.mp4 +2025-01-18 19:12:52,637 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:52,778 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:53,154 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:53,820 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:12:54,112 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:12:54,112 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:12:54,156 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:55,160 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:56,155 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:56,779 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:12:57,161 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:58,089 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:12:58,152 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:12:59,156 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:00,159 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:00,780 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:01,164 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:01,466 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:13:01,467 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:13:01,467 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:13:01,780 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:13:01,780 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:13:02,042 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/mm5z1mqe-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:13:02,163 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:02,781 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:03,175 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:03,225 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:04,171 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:04,781 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:05,166 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:06,171 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:06,782 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:07,167 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:08,168 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:08,662 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:08,783 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:09,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:13:09,082 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:13:09,169 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:10,168 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:11,168 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:12,169 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:13,169 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:14,090 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:14,169 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:15,169 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:16,169 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:17,169 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:18,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:19,090 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:19,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:20,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:20,786 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:21,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:22,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:23,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:23,820 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:13:24,082 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:13:24,082 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:13:24,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:24,236 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:25,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:26,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:26,788 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:27,171 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:28,171 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:29,171 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:30,091 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:30,171 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:31,172 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:32,172 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:33,172 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:34,209 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:34,791 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:34,926 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:13:34,926 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_c150bb925f103adcede9.mp4 with policy now +2025-01-18 19:13:34,946 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:13:34,947 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:13:34,947 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:13:34,947 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:13:35,179 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:35,200 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/agr22d06-media/videos/eval/5_eval_5_c150bb925f103adcede9.mp4 +2025-01-18 19:13:35,791 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:13:35,791 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/5_eval_5_c150bb925f103adcede9.mp4 +2025-01-18 19:13:35,791 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:13:35,994 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:36,112 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/qh0l4mdb-media/videos/eval/5_eval_5_c150bb925f103adcede9.mp4 +2025-01-18 19:13:36,184 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:36,791 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:37,188 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:37,792 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:38,181 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:39,091 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:13:39,091 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:13:39,181 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:40,182 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:40,793 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:41,182 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:41,225 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:42,182 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:42,793 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:43,181 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:44,191 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:44,794 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:13:45,188 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:45,521 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:13:45,521 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:13:45,522 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:13:45,794 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:13:45,794 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:13:46,199 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:46,795 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:46,889 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:47,192 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:48,198 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:49,190 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:50,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:50,796 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:51,189 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:52,168 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:52,189 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:52,796 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:13:53,189 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:53,820 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:13:54,085 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:13:54,085 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:13:54,190 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:55,190 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:56,190 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:57,191 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:58,167 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:13:58,191 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:13:59,191 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:00,195 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:01,193 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:02,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:03,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:03,545 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:04,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:04,800 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:05,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:06,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:07,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:08,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:09,086 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:14:09,086 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:14:09,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:09,261 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:10,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:10,802 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:11,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:12,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:13,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:14,194 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:14,307 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:15,195 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:16,195 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:16,804 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:17,195 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:18,203 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:18,805 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:19,093 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:14:19,093 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_438f8d27cd8ba8c28d6a.mp4 with policy now +2025-01-18 19:14:19,101 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:14:19,102 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:14:19,102 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:14:19,102 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:14:19,198 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:19,364 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/o61o0xqm-media/videos/eval/6_eval_6_438f8d27cd8ba8c28d6a.mp4 +2025-01-18 19:14:19,805 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:14:19,805 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/6_eval_6_438f8d27cd8ba8c28d6a.mp4 +2025-01-18 19:14:19,805 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:14:20,111 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/p9fnoe43-media/videos/eval/6_eval_6_438f8d27cd8ba8c28d6a.mp4 +2025-01-18 19:14:20,157 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:20,216 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:20,805 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:21,212 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:22,204 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:23,221 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:23,821 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:14:24,120 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:14:24,120 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:14:24,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:24,807 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:25,202 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:25,212 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:26,213 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:26,807 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:27,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:28,215 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:29,211 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:30,241 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:30,242 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:30,576 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:14:30,577 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:14:30,577 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:14:30,808 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:14:30,809 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:14:31,054 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/vwd1mvin-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:14:31,217 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:32,217 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:32,809 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:33,219 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:34,217 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:35,217 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:35,558 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:36,217 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:36,811 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:37,217 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:38,217 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:38,811 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:39,086 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:14:39,086 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:14:39,217 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:40,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:41,180 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:41,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:42,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:43,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:44,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:44,813 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:45,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:46,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:46,737 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:46,814 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:47,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:48,218 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:49,219 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:50,219 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:51,219 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:52,098 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:52,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:53,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:53,821 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:14:54,086 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:14:54,087 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:14:54,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:54,817 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:14:55,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:56,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:57,167 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:14:57,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:58,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:14:59,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:00,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:01,221 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:02,225 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:02,347 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:02,397 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:15:02,397 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_d015da1851d414bb6628.mp4 with policy now +2025-01-18 19:15:02,419 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:15:02,420 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:15:02,420 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:15:02,421 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:15:02,668 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/slv61nk6-media/videos/eval/7_eval_7_d015da1851d414bb6628.mp4 +2025-01-18 19:15:02,819 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:02,819 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:15:02,819 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/7_eval_7_d015da1851d414bb6628.mp4 +2025-01-18 19:15:02,820 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:15:03,122 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/dwykm6q5-media/videos/eval/7_eval_7_d015da1851d414bb6628.mp4 +2025-01-18 19:15:03,257 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:03,819 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:04,242 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:04,820 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:05,268 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:06,252 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:07,247 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:08,248 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:08,249 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:08,821 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:09,100 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:15:09,100 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:15:09,247 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:10,263 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:10,822 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:11,265 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:12,275 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:13,266 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:14,126 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:14,268 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:15,273 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:16,268 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:17,268 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:18,271 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:19,272 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:20,114 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:20,273 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:21,270 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:22,286 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:22,642 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:15:22,643 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:15:22,644 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:15:22,825 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:15:22,826 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:22,826 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:15:23,279 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:23,822 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:15:24,093 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:15:24,094 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:15:24,282 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:24,826 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:25,168 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:25,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:26,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:27,292 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:28,280 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:28,827 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:29,295 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:30,252 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:30,281 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:30,828 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:31,281 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:32,282 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:32,829 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:33,281 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:34,282 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:35,282 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:36,108 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:36,282 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:37,282 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:38,282 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:39,091 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:15:39,091 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:15:39,282 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:40,282 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:41,229 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:41,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:42,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:43,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:44,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:45,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:46,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:47,108 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:47,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:48,283 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:49,284 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:50,284 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:51,284 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:52,109 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:52,284 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:53,284 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:53,822 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:15:54,091 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:15:54,091 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:15:54,284 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:55,285 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:56,285 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:57,290 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:57,843 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:15:58,291 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:58,576 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:15:58,576 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_9d16e439b84d1b213c76.mp4 with policy now +2025-01-18 19:15:58,582 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:15:58,582 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:15:58,582 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:15:58,582 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:15:58,848 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:15:58,848 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:15:58,848 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/8_eval_8_9d16e439b84d1b213c76.mp4 +2025-01-18 19:15:58,848 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:15:59,185 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/2v1k81xu-media/videos/eval/8_eval_8_9d16e439b84d1b213c76.mp4 +2025-01-18 19:15:59,298 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:15:59,472 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/r20sy1rb-media/videos/eval/8_eval_8_9d16e439b84d1b213c76.mp4 +2025-01-18 19:15:59,848 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:00,305 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:00,848 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:01,302 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:01,849 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:02,324 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:02,849 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:03,181 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:03,328 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:04,330 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:05,308 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:06,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:06,850 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:07,327 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:08,237 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:08,323 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:09,117 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:16:09,117 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:16:09,331 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:10,329 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:11,326 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:11,705 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:16:11,705 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:16:11,705 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:16:11,852 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:16:11,852 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:16:12,094 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/a6x8cl2s-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:16:12,347 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:12,852 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:13,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:13,397 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:14,350 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:14,853 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:15,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:16,332 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:16,854 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:17,333 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:18,333 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:18,799 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:19,333 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:20,333 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:21,333 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:22,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:23,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:23,823 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:16:23,824 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:24,114 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:16:24,114 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:16:24,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:25,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:26,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:27,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:28,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:29,126 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:29,334 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:30,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:31,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:32,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:33,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:34,127 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:34,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:35,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:36,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:37,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:38,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:39,114 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:16:39,114 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:16:39,200 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:39,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:40,335 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:41,336 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:42,336 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:43,341 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:44,099 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:16:44,099 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_d74f7b5c31b1fe3c2190.mp4 with policy now +2025-01-18 19:16:44,104 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:16:44,105 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:16:44,105 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:16:44,105 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:16:44,344 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:44,359 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/is25lsai-media/videos/eval/9_eval_9_d74f7b5c31b1fe3c2190.mp4 +2025-01-18 19:16:44,863 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:44,863 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:16:44,863 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/9_eval_9_d74f7b5c31b1fe3c2190.mp4 +2025-01-18 19:16:44,864 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:16:45,174 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/drx1gn77-media/videos/eval/9_eval_9_d74f7b5c31b1fe3c2190.mp4 +2025-01-18 19:16:45,190 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:45,360 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:45,863 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:46,353 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:47,377 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:48,353 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:48,864 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:49,354 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:50,363 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:50,695 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:51,360 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:52,357 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:52,866 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:16:53,365 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:53,823 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:16:54,129 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:16:54,129 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:16:54,366 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:55,386 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:56,238 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:16:56,379 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:57,376 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:58,422 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:16:59,419 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:00,424 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:01,419 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:02,150 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:02,419 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:03,426 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:04,422 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:05,423 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:06,426 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:07,153 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:07,448 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:08,449 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:09,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:17:09,143 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:17:09,450 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:10,446 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:11,473 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:11,778 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:17:11,778 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:17:11,778 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:17:11,872 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:17:11,872 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:17:12,354 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:12,478 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:12,873 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:13,470 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:14,471 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:14,874 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:15,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:16,473 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:16,874 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:17,401 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:17,485 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:18,466 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:18,875 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:19,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:20,466 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:20,876 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:21,466 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:22,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:23,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:23,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:23,824 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:17:24,127 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:17:24,128 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:17:24,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:25,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:26,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:27,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:28,203 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:28,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:29,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:30,468 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:31,468 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:32,468 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:33,468 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:34,139 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:34,468 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:35,468 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:36,468 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:37,468 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:38,468 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:38,882 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:39,127 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:17:39,128 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:17:39,240 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:39,469 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:40,469 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:41,469 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:42,470 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:43,470 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:44,470 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:45,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:45,477 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:46,481 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:46,681 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:17:46,681 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_741429cbea5a900591e5.mp4 with policy now +2025-01-18 19:17:46,681 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:17:46,682 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:17:46,682 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:17:46,683 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:17:46,885 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:46,885 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:17:46,885 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/10_eval_10_741429cbea5a900591e5.mp4 +2025-01-18 19:17:46,885 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:17:46,982 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/cyxwk091-media/videos/eval/10_eval_10_741429cbea5a900591e5.mp4 +2025-01-18 19:17:47,277 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/tlr2jk1a-media/videos/eval/10_eval_10_741429cbea5a900591e5.mp4 +2025-01-18 19:17:47,484 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:47,885 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:48,511 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:48,886 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:49,496 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:50,165 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:50,480 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:50,886 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:51,484 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:52,507 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:53,482 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:53,834 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:17:54,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:17:54,142 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:17:54,497 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:54,887 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:17:55,216 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:17:55,502 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:56,507 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:57,509 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:58,523 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:17:59,524 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:00,533 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:01,166 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:01,529 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:02,544 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:03,550 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:04,567 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:05,564 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:06,547 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:07,161 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:07,547 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:08,588 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:09,134 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:18:09,135 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:18:09,560 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:10,559 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:11,577 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:12,208 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:12,560 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:13,570 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:13,860 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:18:13,861 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:18:13,861 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:18:13,894 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:18:13,894 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:18:14,135 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/smvssqfe-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:18:14,621 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:14,894 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:18:15,575 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:16,576 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:16,895 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:18:17,563 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:17,609 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:18,581 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:18,896 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:18:19,582 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:20,582 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:21,582 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:22,582 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:23,155 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:23,582 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:23,824 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:18:24,129 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:18:24,130 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:18:24,582 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:25,582 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:26,582 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:27,583 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:28,267 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:28,583 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:29,583 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:30,583 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:31,583 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:32,583 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:33,584 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:34,155 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:34,584 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:35,584 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:36,584 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:37,584 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:38,584 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:39,129 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:18:39,130 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:18:39,263 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:39,584 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:40,585 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:40,903 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:18:41,588 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:41,757 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:18:41,757 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_5d5f6eccd1ac8a9641c2.mp4 with policy now +2025-01-18 19:18:41,759 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:18:41,760 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:18:41,760 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:18:41,760 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:18:41,904 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:18:41,904 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/11_eval_11_5d5f6eccd1ac8a9641c2.mp4 +2025-01-18 19:18:41,904 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:18:42,014 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/ynl7vygw-media/videos/eval/11_eval_11_5d5f6eccd1ac8a9641c2.mp4 +2025-01-18 19:18:42,310 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/dc8azret-media/videos/eval/11_eval_11_5d5f6eccd1ac8a9641c2.mp4 +2025-01-18 19:18:42,592 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:42,904 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:18:43,611 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:43,904 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:18:44,591 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:45,203 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:45,606 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:46,603 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:46,905 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:18:47,616 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:48,623 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:49,612 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:50,616 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:50,907 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:18:51,163 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:51,617 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:52,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:53,632 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:53,825 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:18:54,162 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:18:54,163 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:18:54,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:55,638 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:56,230 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:18:56,651 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:56,909 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:18:56,909 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:18:56,909 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:18:56,910 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:18:56,910 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:18:57,632 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:57,910 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:18:58,639 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:18:58,910 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:18:59,645 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:00,653 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:01,320 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:01,628 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:02,628 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:02,912 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:03,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:04,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:05,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:06,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:06,813 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:06,914 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:07,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:08,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:09,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:19:09,145 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:19:09,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:10,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:11,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:12,271 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:12,629 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:13,630 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:14,630 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:15,630 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:16,630 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:17,630 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:18,158 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:18,630 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:19,630 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:20,631 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:21,631 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:22,634 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:23,159 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:23,635 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:23,825 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:19:24,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:19:24,145 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:19:24,633 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:24,920 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:25,634 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:26,636 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:27,635 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:28,179 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:28,677 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:28,921 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:29,468 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:19:29,468 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_9c363b6170975b374255.mp4 with policy now +2025-01-18 19:19:29,491 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:19:29,491 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:19:29,492 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:19:29,492 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:19:29,644 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:29,745 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/df1eu0ym-media/videos/eval/12_eval_12_9c363b6170975b374255.mp4 +2025-01-18 19:19:29,922 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:19:29,922 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/12_eval_12_9c363b6170975b374255.mp4 +2025-01-18 19:19:29,922 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:19:30,242 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/pz1b8jun-media/videos/eval/12_eval_12_9c363b6170975b374255.mp4 +2025-01-18 19:19:30,655 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:30,922 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:31,653 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:31,923 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:32,670 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:33,195 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:33,668 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:34,722 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:34,924 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:35,678 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:36,666 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:36,925 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:37,680 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:38,417 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:38,695 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:39,193 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:19:39,194 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:19:39,709 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:39,926 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:19:39,966 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:19:39,966 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:19:39,966 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:19:40,712 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:40,926 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:40,926 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:19:41,708 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:42,732 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:43,477 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:43,700 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:44,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:44,927 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:19:45,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:46,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:47,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:48,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:49,167 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:49,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:50,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:51,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:52,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:53,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:53,826 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:19:54,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:19:54,145 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:19:54,221 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:19:54,699 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:55,699 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:56,699 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:57,699 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:58,699 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:19:59,700 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:00,168 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:00,700 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:01,701 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:02,701 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:03,701 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:04,701 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:05,701 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:06,168 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:06,701 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:07,702 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:08,702 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:09,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:20:09,145 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:20:09,702 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:10,702 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:11,667 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:11,731 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:12,396 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:20:12,396 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_c552251a3cda7642b754.mp4 with policy now +2025-01-18 19:20:12,410 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:20:12,411 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:20:12,411 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:20:12,411 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:20:12,705 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/nn254u81-media/videos/eval/13_eval_13_c552251a3cda7642b754.mp4 +2025-01-18 19:20:12,712 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:12,938 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:20:12,943 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/13_eval_13_c552251a3cda7642b754.mp4 +2025-01-18 19:20:12,943 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:20:13,272 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/4bkggix9-media/videos/eval/13_eval_13_c552251a3cda7642b754.mp4 +2025-01-18 19:20:13,723 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:13,939 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:20:14,725 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:15,765 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:16,754 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:17,153 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:17,789 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:17,940 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:20:18,778 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:19,742 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:20,755 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:21,786 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:21,942 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:20:22,433 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:22,787 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:23,767 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:23,826 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:20:24,166 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:20:24,166 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:20:24,755 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:25,764 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:26,803 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:27,784 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:28,263 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:28,767 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:29,780 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:30,809 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:31,763 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:32,780 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:33,802 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:34,179 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:34,815 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:35,820 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:36,803 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:37,812 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:38,793 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:39,179 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:39,183 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:20:39,183 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:20:39,815 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:40,848 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:41,812 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:42,841 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:43,817 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:44,191 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:44,821 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:45,833 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:46,840 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:47,824 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:48,836 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:49,192 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:49,830 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:50,909 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:51,891 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:52,865 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:53,827 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:20:53,876 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:54,190 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:20:54,191 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:20:54,279 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:20:54,895 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:55,896 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:56,893 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:57,890 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:58,888 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:20:59,898 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:00,210 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:00,895 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:01,908 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:02,922 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:03,919 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:04,956 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:05,924 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:06,202 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:06,927 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:07,922 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:08,943 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:09,213 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:21:09,214 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:21:09,945 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:10,950 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:11,380 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:11,932 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:12,946 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:13,938 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:14,934 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:15,928 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:16,938 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:17,213 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:17,986 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:18,994 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:20,011 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:20,982 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:21,999 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:22,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:23,011 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:23,827 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:21:24,012 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:24,170 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:21:24,170 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:21:25,002 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:26,006 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:26,965 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:21:27,021 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:27,090 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:21:27,090 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:21:27,091 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:21:27,217 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/8z9t98wv-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:21:27,777 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:27,965 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:21:27,965 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:21:28,018 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:29,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:29,966 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:21:30,029 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:31,031 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:32,035 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:32,986 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:33,034 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:33,967 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:21:34,036 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:35,059 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:36,031 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:37,065 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:38,062 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:38,239 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:39,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:39,201 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:21:39,202 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:21:40,036 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:41,036 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:41,970 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:21:42,036 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:43,036 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:43,349 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:44,036 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:45,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:46,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:47,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:48,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:49,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:49,215 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:50,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:51,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:52,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:53,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:53,828 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:21:54,037 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:54,168 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:21:54,168 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:21:54,280 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:21:55,038 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:56,038 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:57,038 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:58,038 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:21:59,038 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:00,038 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:00,216 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:01,039 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:02,039 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:03,039 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:04,039 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:05,039 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:05,965 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:06,040 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:07,045 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:07,306 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:22:07,306 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_58248ea2be38c14379cd.mp4 with policy now +2025-01-18 19:22:07,310 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:22:07,310 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:22:07,310 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:22:07,311 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:22:07,616 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/04mdop80-media/videos/eval/14_eval_14_58248ea2be38c14379cd.mp4 +2025-01-18 19:22:07,980 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:22:07,980 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:22:07,980 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/14_eval_14_58248ea2be38c14379cd.mp4 +2025-01-18 19:22:07,980 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:22:08,044 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:08,337 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/g1xeiq7w-media/videos/eval/14_eval_14_58248ea2be38c14379cd.mp4 +2025-01-18 19:22:09,080 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:09,176 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:22:09,176 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:22:09,980 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:22:10,066 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:10,977 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:11,074 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:12,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:13,074 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:13,982 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:22:14,073 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:15,072 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:15,983 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:22:16,096 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:16,266 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:17,091 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:18,095 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:19,089 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:19,984 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:22:20,098 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:20,156 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:22:20,156 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:22:20,157 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:22:20,984 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:22:20,985 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:22:21,119 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:21,985 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:22:22,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:22,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:23,126 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:23,829 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:22:23,986 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:22:24,118 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:24,181 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:22:24,181 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:22:25,160 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:25,986 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:22:26,130 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:27,132 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:27,413 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:28,154 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:29,153 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:30,162 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:31,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:31,989 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:22:32,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:32,570 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:33,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:33,989 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:22:34,142 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:35,143 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:36,144 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:37,144 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:38,144 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:38,264 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:39,144 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:39,173 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:22:39,173 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:22:40,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:41,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:42,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:43,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:43,264 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:44,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:45,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:46,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:47,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:48,145 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:48,265 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:49,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:50,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:51,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:52,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:53,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:53,829 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:22:53,830 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:54,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:54,174 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:22:54,174 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:22:55,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:56,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:57,146 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:58,157 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:59,074 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:22:59,167 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:22:59,194 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:22:59,194 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_3efd5b9c2b7508acddc5.mp4 with policy now +2025-01-18 19:22:59,237 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:22:59,238 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:22:59,238 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:22:59,240 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:22:59,498 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/abunlzd2-media/videos/eval/15_eval_15_3efd5b9c2b7508acddc5.mp4 +2025-01-18 19:22:59,999 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:00,000 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:23:00,000 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/15_eval_15_3efd5b9c2b7508acddc5.mp4 +2025-01-18 19:23:00,000 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:23:00,210 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:00,379 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/zvjhtqjt-media/videos/eval/15_eval_15_3efd5b9c2b7508acddc5.mp4 +2025-01-18 19:23:01,166 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:02,000 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:02,209 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:03,174 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:04,189 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:04,311 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:05,220 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:06,002 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:06,180 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:07,190 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:08,002 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:08,189 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:09,184 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:23:09,185 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:23:09,225 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:09,322 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:10,192 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:11,190 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:12,215 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:13,202 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:14,216 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:14,323 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:15,214 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:16,005 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:23:16,225 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:23:16,226 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:23:16,226 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:23:16,237 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:17,006 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:23:17,214 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:18,006 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:18,274 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:19,286 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:20,008 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:20,044 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:20,261 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:21,278 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:22,009 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:22,316 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:23,292 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:23,829 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:23:24,211 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:23:24,211 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:23:24,295 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:25,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:25,315 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:26,309 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:27,307 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:28,346 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:29,338 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:30,310 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:30,446 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:31,311 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:32,012 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:32,311 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:33,311 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:34,312 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:35,312 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:36,309 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:36,312 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:37,312 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:38,312 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:39,180 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:23:39,180 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:23:39,313 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:40,313 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:41,313 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:41,331 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:42,313 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:43,313 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:44,313 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:45,313 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:46,313 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:47,309 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:47,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:48,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:49,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:50,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:51,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:52,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:52,379 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:53,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:53,830 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:23:54,022 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:54,180 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:23:54,180 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:23:54,314 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:55,315 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:56,022 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:56,315 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:57,333 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:57,498 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:23:57,787 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:23:57,787 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_a29e67cd0c3e166bf43e.mp4 with policy now +2025-01-18 19:23:57,805 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:23:57,805 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:23:57,806 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:23:57,806 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:23:58,023 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:23:58,023 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:23:58,023 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/16_eval_16_a29e67cd0c3e166bf43e.mp4 +2025-01-18 19:23:58,024 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:23:58,097 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/iw0ep7i3-media/videos/eval/16_eval_16_a29e67cd0c3e166bf43e.mp4 +2025-01-18 19:23:58,321 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:23:58,430 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/v3rfnh24-media/videos/eval/16_eval_16_a29e67cd0c3e166bf43e.mp4 +2025-01-18 19:23:59,332 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:00,024 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:24:00,345 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:01,346 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:02,359 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:03,366 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:03,469 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:04,025 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:24:04,417 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:05,392 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:06,026 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:24:06,404 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:07,389 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:08,411 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:08,758 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:09,211 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:24:09,211 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:24:09,404 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:10,442 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:11,432 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:12,423 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:13,029 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:24:13,279 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/cvqcv4bk-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:24:13,293 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:24:13,293 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:24:13,294 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:24:13,434 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:14,029 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:24:14,029 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:24:14,337 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:14,434 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:15,442 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:16,030 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:24:16,437 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:17,458 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:18,446 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:19,346 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:19,449 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:20,031 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:24:20,462 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:21,458 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:22,456 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:23,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:23,830 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:24:24,216 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:24:24,217 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:24:24,448 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:25,341 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:25,466 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:26,465 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:27,461 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:28,460 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:29,461 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:30,035 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:24:30,461 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:31,330 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:31,461 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:32,461 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:33,462 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:34,462 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:35,462 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:36,330 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:36,462 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:37,462 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:38,463 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:39,201 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:24:39,201 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:24:39,463 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:40,463 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:41,331 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:41,463 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:42,463 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:43,463 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:44,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:45,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:46,331 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:46,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:47,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:48,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:49,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:50,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:51,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:52,331 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:52,465 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:53,464 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:53,831 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:24:54,201 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:24:54,201 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:24:54,465 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:55,540 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:56,045 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:24:56,397 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:24:56,397 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_9f5ca32b8d4315bd9618.mp4 with policy now +2025-01-18 19:24:56,467 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:24:56,467 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:24:56,467 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:24:56,468 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:24:56,517 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:56,709 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/k33lvbg0-media/videos/eval/17_eval_17_9f5ca32b8d4315bd9618.mp4 +2025-01-18 19:24:57,046 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:24:57,046 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/17_eval_17_9f5ca32b8d4315bd9618.mp4 +2025-01-18 19:24:57,046 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:24:57,401 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/3vdpryqj-media/videos/eval/17_eval_17_9f5ca32b8d4315bd9618.mp4 +2025-01-18 19:24:57,478 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:57,621 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:24:58,046 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:24:58,512 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:24:59,537 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:00,509 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:01,548 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:02,048 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:02,543 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:03,319 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:03,523 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:04,539 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:05,526 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:06,049 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:06,550 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:07,552 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:08,363 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:08,551 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:09,224 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:25:09,225 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:25:09,589 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:10,051 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:25:10,363 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:25:10,363 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:25:10,363 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:25:10,615 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:11,051 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:25:11,584 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:12,052 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:12,595 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:13,604 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:14,005 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:14,052 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:14,575 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:15,593 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:16,053 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:16,601 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:17,581 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:18,615 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:19,355 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:19,619 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:20,600 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:21,615 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:22,610 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:23,608 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:23,831 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:25:24,216 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:25:24,216 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:25:24,361 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:24,606 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:25,606 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:26,057 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:26,607 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:27,606 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:28,607 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:29,361 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:29,607 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:30,607 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:31,607 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:32,607 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:33,607 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:34,361 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:34,608 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:35,608 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:36,608 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:37,608 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:38,608 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:39,216 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:25:39,216 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:25:39,361 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:39,608 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:40,608 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:41,609 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:42,609 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:43,609 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:44,361 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:44,609 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:45,609 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:46,609 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:47,631 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:48,628 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:48,824 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:25:48,824 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_fe8dc0d3e79f58b5302a.mp4 with policy now +2025-01-18 19:25:48,834 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:25:48,835 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:25:48,835 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:25:48,835 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:25:49,066 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:25:49,066 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/18_eval_18_fe8dc0d3e79f58b5302a.mp4 +2025-01-18 19:25:49,066 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:25:49,124 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/glcvjgia-media/videos/eval/18_eval_18_fe8dc0d3e79f58b5302a.mp4 +2025-01-18 19:25:49,364 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:49,449 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/8egwbfdo-media/videos/eval/18_eval_18_fe8dc0d3e79f58b5302a.mp4 +2025-01-18 19:25:49,653 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:50,067 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:50,689 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:51,758 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:52,067 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:52,685 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:53,656 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:53,832 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:25:54,068 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:54,255 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:25:54,255 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:25:54,412 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:25:54,662 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:55,713 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:56,717 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:57,709 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:58,070 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:25:58,070 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:25:58,421 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:25:58,422 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:25:58,422 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:25:58,694 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:25:59,070 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:25:59,701 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:00,071 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:26:00,404 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:00,695 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:01,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:02,072 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:26:02,689 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:03,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:04,072 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:26:04,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:05,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:06,388 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:06,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:07,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:08,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:09,221 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:26:09,221 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:26:09,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:10,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:11,388 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:11,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:12,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:13,690 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:14,691 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:15,691 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:16,388 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:16,691 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:17,691 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:18,691 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:19,691 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:20,691 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:21,388 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:21,692 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:22,693 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:23,692 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:23,832 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:26:24,221 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:26:24,221 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:26:24,692 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:25,692 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:26,388 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:26,693 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:27,693 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:28,693 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:29,698 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:30,708 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:30,805 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:26:30,805 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_98200c8b3f44031bae08.mp4 with policy now +2025-01-18 19:26:30,821 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:26:30,821 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:26:30,821 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:30,822 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:31,077 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/5xavx3hz-media/videos/eval/19_eval_19_98200c8b3f44031bae08.mp4 +2025-01-18 19:26:31,083 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:26:31,083 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/19_eval_19_98200c8b3f44031bae08.mp4 +2025-01-18 19:26:31,083 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval +2025-01-18 19:26:31,383 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/dao9cprm-media/videos/eval/19_eval_19_98200c8b3f44031bae08.mp4 +2025-01-18 19:26:31,706 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:31,920 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:32,083 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:26:32,781 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:33,766 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:34,793 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:35,757 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:36,085 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:26:36,793 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:37,697 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:37,783 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:38,781 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:39,087 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:26:39,260 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:26:39,260 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:26:39,471 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:26:39,471 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:39,472 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:39,774 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:40,087 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:26:40,087 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:26:40,087 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:26:40,812 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:41,341 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 19:26:41,341 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 19:26:41,508 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:26:41,774 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:41,802 INFO wandb-upload_0:2557979 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9w7qxo4k +2025-01-18 19:26:41,873 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 19:26:41,880 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:26:42,088 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:26:42,088 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:26:42,088 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/table/eval/success_rate_ci_20_e18b08b27993f2ae79a7.table.json +2025-01-18 19:26:42,089 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/table/eval/total_success_rate_ci_21_41785b785bae78c60d60.table.json +2025-01-18 19:26:42,089 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media +2025-01-18 19:26:42,089 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/table +2025-01-18 19:26:42,089 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/table/eval +2025-01-18 19:26:42,164 INFO SenderThread:2557979 [sender.py:send_request_log_artifact():1525] logged artifact run-u3lo36eq-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTg4ODA1MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzAyNTMyMQ==', 'latestArtifact': None}} +2025-01-18 19:26:42,164 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:26:42,164 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_e18b08b27993f2ae79a7.table.json with policy now +2025-01-18 19:26:42,165 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:26:42,165 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,165 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,165 DEBUG SenderThread:2557979 [sender.py:send():379] send: telemetry +2025-01-18 19:26:42,165 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,169 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,170 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,170 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,170 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,170 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,171 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,171 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,172 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,172 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,172 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,172 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,173 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,173 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,173 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,174 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,174 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,174 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,174 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,175 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,175 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:42,175 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:42,175 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 19:26:42,381 INFO wandb-upload_2:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/c9ow2w6m-media/table/eval/total_success_rate_ci_21_41785b785bae78c60d60.table.json +2025-01-18 19:26:42,402 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/0ozp5w34-media/table/eval/success_rate_ci_20_e18b08b27993f2ae79a7.table.json +2025-01-18 19:26:42,411 INFO wandb-upload_1:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/jaeudeso-events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:26:42,611 INFO wandb-upload_2:2557979 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9ffc0x7r +2025-01-18 19:26:42,680 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/50fdirc7-media/table/eval/success_rate_ci_20_e18b08b27993f2ae79a7.table.json +2025-01-18 19:26:42,774 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:43,001 INFO SenderThread:2557979 [sender.py:send_request_log_artifact():1525] logged artifact run-u3lo36eq-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTg4ODA2Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzAyNTMyNg==', 'latestArtifact': None}} +2025-01-18 19:26:43,004 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:43,117 DEBUG SenderThread:2557979 [sender.py:send():379] send: files +2025-01-18 19:26:43,118 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_41785b785bae78c60d60.table.json with policy now +2025-01-18 19:26:43,118 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:26:43,118 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:26:43,118 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:43,119 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:26:43,370 INFO wandb-upload_1:2557979 [upload_job.py:push():130] Uploaded file /tmp/tmpmus_h_z9wandb/3k67plp1-media/table/eval/total_success_rate_ci_21_41785b785bae78c60d60.table.json +2025-01-18 19:26:43,774 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:44,120 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/config.yaml +2025-01-18 19:26:44,120 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:26:44,120 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:26:44,775 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:45,775 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:46,775 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:47,775 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:48,390 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:48,775 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:49,775 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:50,775 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:51,775 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:52,775 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:53,776 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:53,832 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:26:53,834 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:26:54,242 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:26:54,242 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:26:54,776 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:55,776 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:56,776 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:57,776 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:26:58,117 DEBUG SenderThread:2557979 [sender.py:send():379] send: exit +2025-01-18 19:26:58,117 INFO SenderThread:2557979 [sender.py:send_exit():586] handling exit code: 0 +2025-01-18 19:26:58,118 INFO SenderThread:2557979 [sender.py:send_exit():588] handling runtime: 1084 +2025-01-18 19:26:58,118 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:26:58,118 INFO SenderThread:2557979 [sender.py:send_exit():594] send defer +2025-01-18 19:26:58,118 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:26:58,118 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-18 19:26:58,118 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:26:58,118 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-18 19:26:58,118 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 1 +2025-01-18 19:26:58,118 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:26:58,119 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-18 19:26:58,119 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:26:58,119 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-18 19:26:58,119 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 2 +2025-01-18 19:26:58,119 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:26:58,119 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-18 19:26:58,119 INFO HandlerThread:2557979 [system_monitor.py:finish():203] Stopping system monitor +2025-01-18 19:26:58,119 DEBUG SystemMonitor:2557979 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-18 19:26:58,119 DEBUG SystemMonitor:2557979 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-18 19:26:58,120 INFO HandlerThread:2557979 [interfaces.py:finish():200] Joined cpu monitor +2025-01-18 19:26:58,120 INFO HandlerThread:2557979 [interfaces.py:finish():200] Joined disk monitor +2025-01-18 19:26:58,127 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:26:58,170 INFO HandlerThread:2557979 [interfaces.py:finish():200] Joined gpu monitor +2025-01-18 19:26:58,170 INFO HandlerThread:2557979 [interfaces.py:finish():200] Joined memory monitor +2025-01-18 19:26:58,170 INFO HandlerThread:2557979 [interfaces.py:finish():200] Joined network monitor +2025-01-18 19:26:58,171 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:26:58,171 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-18 19:26:58,171 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 3 +2025-01-18 19:26:58,171 DEBUG SenderThread:2557979 [sender.py:send():379] send: stats +2025-01-18 19:26:58,171 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:26:58,171 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-18 19:26:58,172 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:26:58,172 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-18 19:26:58,172 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 4 +2025-01-18 19:26:58,172 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:26:58,172 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-18 19:27:00,128 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:27:04,478 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:27:04,479 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:04,479 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-18 19:27:04,479 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 5 +2025-01-18 19:27:04,479 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:27:04,479 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:27:04,479 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: keepalive +2025-01-18 19:27:04,479 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:27:04,480 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:04,480 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-18 19:27:04,481 DEBUG SenderThread:2557979 [sender.py:send():379] send: history +2025-01-18 19:27:04,481 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:27:04,481 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:27:04,482 DEBUG SenderThread:2557979 [sender.py:send():379] send: summary +2025-01-18 19:27:04,486 INFO SenderThread:2557979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:27:04,486 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:04,486 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-18 19:27:04,486 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 6 +2025-01-18 19:27:04,486 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:04,486 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-18 19:27:04,486 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:04,486 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-18 19:27:04,486 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 7 +2025-01-18 19:27:04,486 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:27:04,486 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:04,487 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-18 19:27:04,487 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:04,487 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-18 19:27:05,119 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:27:05,130 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:27:05,746 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 8 +2025-01-18 19:27:05,746 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:27:05,746 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:05,746 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-18 19:27:05,746 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:05,746 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-18 19:27:05,746 INFO SenderThread:2557979 [job_builder.py:build():440] Attempting to build job artifact +2025-01-18 19:27:05,749 INFO SenderThread:2557979 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-18 19:27:05,751 INFO SenderThread:2557979 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-18 19:27:05,765 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 9 +2025-01-18 19:27:05,765 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:05,765 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-18 19:27:05,765 DEBUG SenderThread:2557979 [sender.py:send():379] send: artifact +2025-01-18 19:27:06,119 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:27:06,130 INFO Thread-12 :2557979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:27:06,204 INFO wandb-upload_2:2557979 [upload_job.py:push():85] Skipped uploading /tmp/tmp1_2fxjxp/wandb-job.json +2025-01-18 19:27:06,204 INFO wandb-upload_0:2557979 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpavoy7qkt +2025-01-18 19:27:06,870 INFO SenderThread:2557979 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1MTg4ODMzOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1MTg1ODU1Ng==', 'versionIndex': 446}}} +2025-01-18 19:27:06,870 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:06,870 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-18 19:27:06,870 INFO SenderThread:2557979 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-18 19:27:07,131 INFO SenderThread:2557979 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files +2025-01-18 19:27:07,131 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/config.yaml config.yaml +2025-01-18 19:27:07,131 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 events.out.tfevents.1737245334.leonmkim-ROG-Strix-G15CS-G15CS.2557788.0 +2025-01-18 19:27:07,133 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json wandb-summary.json +2025-01-18 19:27:07,133 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/requirements.txt requirements.txt +2025-01-18 19:27:07,135 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-metadata.json wandb-metadata.json +2025-01-18 19:27:07,135 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log output.log +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/12_eval_12_9c363b6170975b374255.mp4 media/videos/eval/12_eval_12_9c363b6170975b374255.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/13_eval_13_c552251a3cda7642b754.mp4 media/videos/eval/13_eval_13_c552251a3cda7642b754.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/16_eval_16_a29e67cd0c3e166bf43e.mp4 media/videos/eval/16_eval_16_a29e67cd0c3e166bf43e.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/4_eval_4_edd1b447fab3432974b4.mp4 media/videos/eval/4_eval_4_edd1b447fab3432974b4.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/18_eval_18_fe8dc0d3e79f58b5302a.mp4 media/videos/eval/18_eval_18_fe8dc0d3e79f58b5302a.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/11_eval_11_5d5f6eccd1ac8a9641c2.mp4 media/videos/eval/11_eval_11_5d5f6eccd1ac8a9641c2.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/7_eval_7_d015da1851d414bb6628.mp4 media/videos/eval/7_eval_7_d015da1851d414bb6628.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/9_eval_9_d74f7b5c31b1fe3c2190.mp4 media/videos/eval/9_eval_9_d74f7b5c31b1fe3c2190.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/19_eval_19_98200c8b3f44031bae08.mp4 media/videos/eval/19_eval_19_98200c8b3f44031bae08.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/0_eval_0_f2d24d1f27d7648055dc.mp4 media/videos/eval/0_eval_0_f2d24d1f27d7648055dc.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/10_eval_10_741429cbea5a900591e5.mp4 media/videos/eval/10_eval_10_741429cbea5a900591e5.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/8_eval_8_9d16e439b84d1b213c76.mp4 media/videos/eval/8_eval_8_9d16e439b84d1b213c76.mp4 +2025-01-18 19:27:07,139 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/14_eval_14_58248ea2be38c14379cd.mp4 media/videos/eval/14_eval_14_58248ea2be38c14379cd.mp4 +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/15_eval_15_3efd5b9c2b7508acddc5.mp4 media/videos/eval/15_eval_15_3efd5b9c2b7508acddc5.mp4 +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/5_eval_5_c150bb925f103adcede9.mp4 media/videos/eval/5_eval_5_c150bb925f103adcede9.mp4 +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/1_eval_1_f85e4baf2746d94aa0a7.mp4 media/videos/eval/1_eval_1_f85e4baf2746d94aa0a7.mp4 +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/2_eval_2_340d87e376b6dfa888fb.mp4 media/videos/eval/2_eval_2_340d87e376b6dfa888fb.mp4 +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/17_eval_17_9f5ca32b8d4315bd9618.mp4 media/videos/eval/17_eval_17_9f5ca32b8d4315bd9618.mp4 +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/3_eval_3_a45dc41fdb9f639ccef4.mp4 media/videos/eval/3_eval_3_a45dc41fdb9f639ccef4.mp4 +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/videos/eval/6_eval_6_438f8d27cd8ba8c28d6a.mp4 media/videos/eval/6_eval_6_438f8d27cd8ba8c28d6a.mp4 +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/table/eval/total_success_rate_ci_21_41785b785bae78c60d60.table.json media/table/eval/total_success_rate_ci_21_41785b785bae78c60d60.table.json +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/media/table/eval/success_rate_ci_20_e18b08b27993f2ae79a7.table.json media/table/eval/success_rate_ci_20_e18b08b27993f2ae79a7.table.json +2025-01-18 19:27:07,140 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 10 +2025-01-18 19:27:07,140 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:27:07,140 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:07,141 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-18 19:27:07,141 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:07,141 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-18 19:27:07,141 INFO SenderThread:2557979 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 19:27:07,308 INFO wandb-upload_2:2557979 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/requirements.txt +2025-01-18 19:27:07,327 INFO wandb-upload_3:2557979 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/output.log +2025-01-18 19:27:07,348 INFO wandb-upload_1:2557979 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/config.yaml +2025-01-18 19:27:07,349 INFO wandb-upload_0:2557979 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/files/wandb-summary.json +2025-01-18 19:27:07,549 INFO Thread-11 (_thread_body):2557979 [sender.py:transition_state():614] send defer: 11 +2025-01-18 19:27:07,549 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:07,549 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-18 19:27:07,549 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:07,549 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-18 19:27:07,549 INFO SenderThread:2557979 [file_pusher.py:join():175] waiting for file pusher +2025-01-18 19:27:07,552 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 12 +2025-01-18 19:27:07,553 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:07,553 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-18 19:27:07,553 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:07,553 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-18 19:27:07,553 INFO SenderThread:2557979 [file_stream.py:finish():601] file stream finish called +2025-01-18 19:27:07,755 INFO SenderThread:2557979 [file_stream.py:finish():605] file stream finish is done +2025-01-18 19:27:07,755 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 13 +2025-01-18 19:27:07,755 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:07,755 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-18 19:27:07,755 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:07,755 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-18 19:27:07,755 INFO SenderThread:2557979 [sender.py:transition_state():614] send defer: 14 +2025-01-18 19:27:07,755 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:27:07,755 INFO HandlerThread:2557979 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-18 19:27:07,756 DEBUG SenderThread:2557979 [sender.py:send():379] send: final +2025-01-18 19:27:07,756 DEBUG SenderThread:2557979 [sender.py:send():379] send: footer +2025-01-18 19:27:07,756 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: defer +2025-01-18 19:27:07,756 INFO SenderThread:2557979 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-18 19:27:07,756 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:27:07,756 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: server_info +2025-01-18 19:27:07,756 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:27:07,757 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: server_info +2025-01-18 19:27:07,758 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:27:07,758 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: get_summary +2025-01-18 19:27:07,759 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-18 19:27:07,760 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:27:07,816 DEBUG SenderThread:2557979 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:27:07,816 INFO MainThread:2557979 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-18 19:27:07,817 INFO MainThread:2557979 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-18 19:27:07,817 INFO MainThread:2557979 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-18 19:27:07,817 DEBUG HandlerThread:2557979 [handler.py:handle_request():158] handle_request: shutdown +2025-01-18 19:27:07,818 INFO HandlerThread:2557979 [handler.py:finish():882] shutting down handler +2025-01-18 19:27:08,758 INFO WriterThread:2557979 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/run-u3lo36eq.wandb +2025-01-18 19:27:08,816 INFO SenderThread:2557979 [sender.py:finish():1615] shutting down sender +2025-01-18 19:27:08,816 INFO SenderThread:2557979 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 19:27:08,816 INFO SenderThread:2557979 [file_pusher.py:join():175] waiting for file pusher diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/logs/debug.log b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0ae07b57eb02879529824bfbcb4b675a932b0bc3 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_setup.py:_flush():76] Configure stats pid to 2557788 +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/settings +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/logs/debug.log +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/logs/debug-internal.log +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_init.py:init():569] calling init triggers +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_1', 'wandb_notes': '3465_1_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq'} +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_init.py:init():619] starting backend +2025-01-18 19:08:53,476 INFO MainThread:2557788 [wandb_init.py:init():623] setting up manager +2025-01-18 19:08:53,482 INFO MainThread:2557788 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-18 19:08:53,483 INFO MainThread:2557788 [wandb_init.py:init():631] backend started and connected +2025-01-18 19:08:53,495 INFO MainThread:2557788 [wandb_init.py:init():720] updated telemetry +2025-01-18 19:08:53,501 INFO MainThread:2557788 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-18 19:08:53,730 INFO MainThread:2557788 [wandb_run.py:_on_init():2435] communicating current version +2025-01-18 19:08:53,802 INFO MainThread:2557788 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-18 19:08:53,802 INFO MainThread:2557788 [wandb_init.py:init():804] starting run threads in backend +2025-01-18 19:08:54,076 INFO MainThread:2557788 [wandb_run.py:_console_start():2413] atexit reg +2025-01-18 19:08:54,076 INFO MainThread:2557788 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-18 19:08:54,077 INFO MainThread:2557788 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-18 19:08:54,077 INFO MainThread:2557788 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-18 19:08:54,078 INFO MainThread:2557788 [wandb_init.py:init():847] run started, returning control to user process +2025-01-18 19:08:54,079 INFO MainThread:2557788 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/u3lo36eq/tb, True +2025-01-18 19:08:58,120 INFO MainThread:2557788 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-18 19:27:08,818 WARNING MsgRouterThr:2557788 [router.py:message_loop():77] message_loop has been closed diff --git a/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/run-u3lo36eq.wandb b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/run-u3lo36eq.wandb new file mode 100644 index 0000000000000000000000000000000000000000..59f64260306fc393f6a4c9a57669f5f0f0ac09a6 --- /dev/null +++ b/u3lo36eq/wandb/run-20250118_190853-u3lo36eq/run-u3lo36eq.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab5a9d04ba4e856c5aff07394196fff9de2755ffbe0152747297f41284eb2eae +size 245638 diff --git a/y2e6goox/.hydra/config.yaml b/y2e6goox/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e01202c5bb687b2213c48829aa3d8f422bf0d242 --- /dev/null +++ b/y2e6goox/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '3465_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/y2e6goox/.hydra/hydra.yaml b/y2e6goox/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4f6ca5f9ab850192963608895320a3d5badaf61 --- /dev/null +++ b/y2e6goox/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_1/y2e6goox + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/y2e6goox/.hydra/overrides.yaml b/y2e6goox/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/y2e6goox/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/y2e6goox/eval_policy.log b/y2e6goox/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..9ac2998b2892af78a81a7856e9a56489e121bbcb --- /dev/null +++ b/y2e6goox/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-18 19:08:40,372][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:429: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-18 19:08:40,375][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:366: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-18 19:08:42,507][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:415: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) +