diff --git a/.gitattributes b/.gitattributes index 592237fa9d042f0e058d4a100de9b4c68d82e9a6..5a4d8ca3b457d7c75b3ff12c99f18844ae735627 100644 --- a/.gitattributes +++ b/.gitattributes @@ -211,3 +211,88 @@ vfeb54wi/episode_rosbags/episode_8_2025-01-15-21-32-40.bag filter=lfs diff=lfs m odxnffav/episode_rosbags/episode_2_2025-01-17-14-31-51.bag filter=lfs diff=lfs merge=lfs -text odxnffav/episode_rosbags/episode_13_2025-01-17-14-39-56.bag filter=lfs diff=lfs merge=lfs -text vfeb54wi/episode_rosbags/episode_0_2025-01-15-21-26-31.bag filter=lfs diff=lfs merge=lfs -text +roatk2bk/episode_rosbags/episode_17_2025-01-15-22-02-19.bag filter=lfs diff=lfs merge=lfs -text +roatk2bk/episode_rosbags/episode_9_2025-01-15-21-52-35.bag filter=lfs diff=lfs merge=lfs -text +odxnffav/episode_rosbags/episode_14_2025-01-17-14-40-42.bag filter=lfs diff=lfs merge=lfs -text +roatk2bk/episode_rosbags/episode_15_2025-01-15-22-00-06.bag filter=lfs diff=lfs merge=lfs -text +roatk2bk/episode_rosbags/episode_10_2025-01-15-21-53-06.bag filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/run-roatk2bk.wandb filter=lfs diff=lfs merge=lfs -text +roatk2bk/episode_rosbags/episode_4_2025-01-15-21-48-45.bag filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/1_eval_1_b5d7870b7b04a6ef485f.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/2_eval_2_f98c31cd2e05324cffd8.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/12_eval_12_241be90b3f1c54884e79.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/15_eval_15_76fa4e81e9e73fef9cf0.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/16_eval_16_5019757f76965e10782b.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/8_eval_8_fb8f14c1c353cc80983a.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/5_eval_5_a69fcdfd49e2066116b6.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/0_eval_0_2eb6cf19687a4fabfe3c.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/episode_rosbags/episode_0_2025-01-15-21-46-03.bag filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/6_eval_6_ae418a6c99a51b4201d6.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/10_eval_10_6e43fe2801f3a905e582.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/14_eval_14_074bdfc80a4a9d765f98.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/18_eval_18_c8c84019c8dc8ba0e4ea.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/17_eval_17_2f96b121fd6f99c60674.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/3_eval_3_89821fb09b372630838b.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/4_eval_4_9fb52356d0a213033cfe.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/13_eval_13_cb3449f02cfc7cf55b8d.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/7_eval_7_f02cea2dbceefebb3b97.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/9_eval_9_19795df8831a10c10837.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/19_eval_19_3f737e19fed74dd298b2.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/11_eval_11_4263c7dc48b07a1c552c.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/19_eval.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/8_eval.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/12_eval.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/7_eval.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/13_eval.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/18_eval.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/9_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/episode_rosbags/episode_12_2025-01-17-14-39-07.bag filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/11_eval.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/15_eval.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/6_eval.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/5_eval.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/16_eval.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/10_eval.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/14_eval.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/eval_video/17_eval.mp4 filter=lfs diff=lfs merge=lfs -text +odxnffav/episode_rosbags/episode_9_2025-01-17-14-36-55.bag filter=lfs diff=lfs merge=lfs -text +odxnffav/episode_rosbags/episode_7_2025-01-17-14-35-26.bag filter=lfs diff=lfs merge=lfs -text +roatk2bk/episode_rosbags/episode_5_2025-01-15-21-49-16.bag filter=lfs diff=lfs merge=lfs -text +roatk2bk/episode_rosbags/episode_11_2025-01-15-21-53-40.bag filter=lfs diff=lfs merge=lfs -text +roatk2bk/episode_rosbags/episode_14_2025-01-15-21-56-03.bag filter=lfs diff=lfs merge=lfs -text +roatk2bk/episode_rosbags/episode_8_2025-01-15-21-51-45.bag filter=lfs diff=lfs merge=lfs -text +odxnffav/episode_rosbags/episode_0_2025-01-17-14-30-16.bag filter=lfs diff=lfs merge=lfs -text +roatk2bk/episode_rosbags/episode_2_2025-01-15-21-47-30.bag filter=lfs diff=lfs merge=lfs -text +roatk2bk/episode_rosbags/episode_18_2025-01-15-22-03-02.bag filter=lfs diff=lfs merge=lfs -text +h18xe90d/wandb/run-20250117_151044-h18xe90d/run-h18xe90d.wandb filter=lfs diff=lfs merge=lfs -text +h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/1_eval_1_fe99acb9f5fbe6b6c23e.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/7_eval_7_eab2215188c302c460ba.mp4 filter=lfs diff=lfs merge=lfs -text +roatk2bk/episode_rosbags/episode_19_2025-01-15-22-03-57.bag filter=lfs diff=lfs merge=lfs -text +h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/0_eval_0_98c68d418db5b73df1b0.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/9_eval_9_9a2233becb7986a47e63.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/4_eval_4_8487a970815421048be9.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/3_eval_3_2988eff819b475f13199.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/11_eval_11_f83167404e8bb0859778.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/6_eval_6_5f3fdb589a9c504e4946.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/10_eval_10_dc26db5ffbb288ebf69b.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/5_eval_5_5eccb1be08111ca14e10.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/8_eval_8_50d0e7e81a609c529244.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/2_eval_2_a9b669c92c8beee6656c.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/eval_video/8_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/eval_video/7_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/episode_rosbags/episode_6_2025-01-17-15-16-02.bag filter=lfs diff=lfs merge=lfs -text +h18xe90d/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/eval_video/9_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/eval_video/11_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/eval_video/6_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/eval_video/5_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/eval_video/10_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h18xe90d/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/h18xe90d/episode_rosbags/aligned_depth_to_color_K.npy b/h18xe90d/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/h18xe90d/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/h18xe90d/episode_rosbags/cam_tf_world.npy b/h18xe90d/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/h18xe90d/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/h18xe90d/episode_rosbags/color_K.npy b/h18xe90d/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/h18xe90d/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/h18xe90d/episode_rosbags/depth_K.npy b/h18xe90d/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/h18xe90d/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/h18xe90d/episode_rosbags/episode_6_2025-01-17-15-16-02.bag b/h18xe90d/episode_rosbags/episode_6_2025-01-17-15-16-02.bag new file mode 100644 index 0000000000000000000000000000000000000000..341dc23b74065481a45b0b15fb32b2f3467856f9 --- /dev/null +++ b/h18xe90d/episode_rosbags/episode_6_2025-01-17-15-16-02.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bd62c575d83d7c558fc62f5f0c78c1d923a04ef464e7afe4fb90b8b1c0a745a +size 1412802434 diff --git a/h18xe90d/eval_robot.log b/h18xe90d/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..0f9514e49ad8532db9e0694aecfda9fe6aefcb5b --- /dev/null +++ b/h18xe90d/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-17 15:10:38,524][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-17 15:10:38,527][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/h18xe90d/eval_video/0_eval.mp4 b/h18xe90d/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..64c693609e2342d36a0cd2a7c8616844e0f1fffb --- /dev/null +++ b/h18xe90d/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98c68d418db5b73df1b05761452a845f5a35493133d7805bbed1dc89315be5c9 +size 1167172 diff --git a/h18xe90d/eval_video/10_eval.mp4 b/h18xe90d/eval_video/10_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0a260c3acaed818b2d422f24013ddd358abb40a4 --- /dev/null +++ b/h18xe90d/eval_video/10_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc26db5ffbb288ebf69be3b37f1a149b05a67b55add127df99546db602a0c6ce +size 1385355 diff --git a/h18xe90d/eval_video/11_eval.mp4 b/h18xe90d/eval_video/11_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3fac08fcc76564381b2f34b9547dd443767d1db8 --- /dev/null +++ b/h18xe90d/eval_video/11_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f83167404e8bb0859778ab785b3ec72f319922e32ba8be47628685d52a3fb45e +size 1451754 diff --git a/h18xe90d/eval_video/12_eval.mp4 b/h18xe90d/eval_video/12_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa868fbb75c51fb59ae42ff30716054804074343 Binary files /dev/null and b/h18xe90d/eval_video/12_eval.mp4 differ diff --git a/h18xe90d/eval_video/1_eval.mp4 b/h18xe90d/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..34ad75c54a1653fae4b114452d1c514f6ed4ba4e --- /dev/null +++ b/h18xe90d/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe99acb9f5fbe6b6c23e6a1f518986a970f45769de1d4acd955b9b5f9bb5e789 +size 1154808 diff --git a/h18xe90d/eval_video/2_eval.mp4 b/h18xe90d/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..520b3d7d70b24559080ec2d1f13db217919ded5e --- /dev/null +++ b/h18xe90d/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9b669c92c8beee6656cacd11f615b7c5aee743d336e4b927a08eddfab14d19b +size 1163094 diff --git a/h18xe90d/eval_video/3_eval.mp4 b/h18xe90d/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9f53218ead20944215e79392c1f28fb1a356bfa0 --- /dev/null +++ b/h18xe90d/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2988eff819b475f131997a6481c4c4ca6d5d6a9124cea83e7998596e71612259 +size 1160636 diff --git a/h18xe90d/eval_video/4_eval.mp4 b/h18xe90d/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cd10d13f869460cd7ad5876af3d21b2804f258b7 --- /dev/null +++ b/h18xe90d/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8487a970815421048be960754b4caeed67e948d55be1d9121a1fb0628f53fb10 +size 1171773 diff --git a/h18xe90d/eval_video/5_eval.mp4 b/h18xe90d/eval_video/5_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..319a5d82563cfbfa0ac652124a2a0660e640e2d5 --- /dev/null +++ b/h18xe90d/eval_video/5_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5eccb1be08111ca14e10f50c8b05a5c13c37a164fba6aaa66d39310d8cfd12f3 +size 1269853 diff --git a/h18xe90d/eval_video/6_eval.mp4 b/h18xe90d/eval_video/6_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2226ee9574a1583ee0291f3b484e7bdcc7facfad --- /dev/null +++ b/h18xe90d/eval_video/6_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f3fdb589a9c504e49465abf5578b1e48353445b5fb7da025d1597ab32b5a0b7 +size 1231149 diff --git a/h18xe90d/eval_video/7_eval.mp4 b/h18xe90d/eval_video/7_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df5ecd54236202a99e4f21523217aca7b73a21e7 --- /dev/null +++ b/h18xe90d/eval_video/7_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eab2215188c302c460ba00e7f817b9602d979b290be42c792b2cff46a93ae595 +size 1153200 diff --git a/h18xe90d/eval_video/8_eval.mp4 b/h18xe90d/eval_video/8_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..015f40bfeb7f352813b6a1f3479ea850141af1b1 --- /dev/null +++ b/h18xe90d/eval_video/8_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50d0e7e81a609c529244b4bf0955c640353908a3899746c56f1df81d8ea406b8 +size 1174619 diff --git a/h18xe90d/eval_video/9_eval.mp4 b/h18xe90d/eval_video/9_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6f859937d7908d50dc10a366080f5f19e086083c --- /dev/null +++ b/h18xe90d/eval_video/9_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a2233becb7986a47e63b28b21334780efa49c26963ca4e624fb1db5d670d747 +size 1362409 diff --git a/h18xe90d/tb/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 b/h18xe90d/tb/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 new file mode 100644 index 0000000000000000000000000000000000000000..b7c1b21b21e0edb6943477c9d2b01c8c4b5d2f78 --- /dev/null +++ b/h18xe90d/tb/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c41bcbd73dc6edffa7bba0f3e5b86a17ebd48f081701fb27475a6d0aa8b40c38 +size 12001 diff --git a/h18xe90d/wandb/debug-internal.log b/h18xe90d/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..fde8e41a65f3a1b66c1349dca5ab6d5c101c296b --- /dev/null +++ b/h18xe90d/wandb/debug-internal.log @@ -0,0 +1,4097 @@ +2025-01-17 15:10:44,431 INFO StreamThr :2520816 [internal.py:wandb_internal():85] W&B internal server running at pid: 2520816, started at: 2025-01-17 15:10:44.431316 +2025-01-17 15:10:44,432 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status +2025-01-17 15:10:44,448 INFO WriterThread:2520816 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/run-h18xe90d.wandb +2025-01-17 15:10:44,448 DEBUG SenderThread:2520816 [sender.py:send():379] send: header +2025-01-17 15:10:44,457 DEBUG SenderThread:2520816 [sender.py:send():379] send: run +2025-01-17 15:10:44,741 INFO SenderThread:2520816 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files +2025-01-17 15:10:44,741 INFO SenderThread:2520816 [sender.py:_start_run_threads():1188] run started: h18xe90d with start time 1737144644.432999 +2025-01-17 15:10:44,754 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: check_version +2025-01-17 15:10:44,755 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: check_version +2025-01-17 15:10:44,802 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: run_start +2025-01-17 15:10:44,811 DEBUG HandlerThread:2520816 [system_info.py:__init__():26] System info init +2025-01-17 15:10:44,811 DEBUG HandlerThread:2520816 [system_info.py:__init__():41] System info init done +2025-01-17 15:10:44,811 INFO HandlerThread:2520816 [system_monitor.py:start():194] Starting system monitor +2025-01-17 15:10:44,811 INFO SystemMonitor:2520816 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-17 15:10:44,812 INFO SystemMonitor:2520816 [interfaces.py:start():188] Started cpu monitoring +2025-01-17 15:10:44,812 INFO HandlerThread:2520816 [system_monitor.py:probe():214] Collecting system info +2025-01-17 15:10:44,813 INFO SystemMonitor:2520816 [interfaces.py:start():188] Started disk monitoring +2025-01-17 15:10:44,814 INFO SystemMonitor:2520816 [interfaces.py:start():188] Started gpu monitoring +2025-01-17 15:10:44,814 INFO SystemMonitor:2520816 [interfaces.py:start():188] Started memory monitoring +2025-01-17 15:10:44,815 INFO SystemMonitor:2520816 [interfaces.py:start():188] Started network monitoring +2025-01-17 15:10:44,824 DEBUG HandlerThread:2520816 [system_info.py:probe():152] Probing system +2025-01-17 15:10:44,826 DEBUG HandlerThread:2520816 [system_info.py:_probe_git():137] Probing git +2025-01-17 15:10:44,832 DEBUG HandlerThread:2520816 [system_info.py:_probe_git():145] Probing git done +2025-01-17 15:10:44,832 DEBUG HandlerThread:2520816 [system_info.py:probe():200] Probing system done +2025-01-17 15:10:44,832 DEBUG HandlerThread:2520816 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-17T20:10:44.824913', 'startedAt': '2025-01-17T20:10:44.420748', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3952.85125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4530.086, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 426.36322021484375}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-17 15:10:44,832 INFO HandlerThread:2520816 [system_monitor.py:probe():224] Finished collecting system info +2025-01-17 15:10:44,832 INFO HandlerThread:2520816 [system_monitor.py:probe():227] Publishing system info +2025-01-17 15:10:44,832 DEBUG HandlerThread:2520816 [system_info.py:_save_code():44] Saving code +2025-01-17 15:10:44,834 DEBUG HandlerThread:2520816 [system_info.py:_save_code():67] Saving code done +2025-01-17 15:10:44,834 DEBUG HandlerThread:2520816 [system_info.py:_save_patches():84] Saving git patches +2025-01-17 15:10:44,934 DEBUG HandlerThread:2520816 [system_info.py:_save_patches():126] Saving git patches done +2025-01-17 15:10:44,934 INFO HandlerThread:2520816 [system_monitor.py:probe():229] Finished publishing system info +2025-01-17 15:10:44,937 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:10:44,937 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-17 15:10:44,938 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-17 15:10:44,938 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-17 15:10:44,938 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-17 15:10:45,170 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: python_packages +2025-01-17 15:10:45,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:10:45,171 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: python_packages +2025-01-17 15:10:45,171 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:10:45,173 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:45,173 INFO HandlerThread:2520816 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/tb" + save: true +} +control { + relay_id: "460f5c04bdcc4fc69d0b7b404204a8cd" +} +_info { + stream_id: "h18xe90d" +} + +2025-01-17 15:10:45,173 DEBUG HandlerThread:2520816 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-17 15:10:45,201 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/0d215zcv-diff.patch +2025-01-17 15:10:45,219 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/ikt77rrz-wandb-metadata.json +2025-01-17 15:10:45,233 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/qydih3fq-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 15:10:45,255 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/be50glop-code/FISH/eval_robot.py +2025-01-17 15:10:45,310 DEBUG SenderThread:2520816 [sender.py:send():379] send: telemetry +2025-01-17 15:10:45,310 DEBUG SenderThread:2520816 [sender.py:send():379] send: tbrecord +2025-01-17 15:10:45,311 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:10:45,311 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 with policy live +2025-01-17 15:10:45,504 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/sc2eh3xv-events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:10:45,742 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:10:45,743 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/code/FISH/eval_robot.py +2025-01-17 15:10:45,743 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 15:10:45,743 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/requirements.txt +2025-01-17 15:10:45,743 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/diff.patch +2025-01-17 15:10:45,743 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-metadata.json +2025-01-17 15:10:45,743 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:10:45,743 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/code/FISH +2025-01-17 15:10:45,743 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/code +2025-01-17 15:10:46,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:47,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:47,742 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:10:48,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:49,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:50,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:10:50,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:51,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:51,874 DEBUG SenderThread:2520816 [sender.py:send():379] send: config +2025-01-17 15:10:52,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:53,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:53,743 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:10:54,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:55,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:10:55,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:55,744 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:10:56,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:57,173 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:57,745 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:10:58,173 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:59,173 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:00,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:11:00,171 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:11:00,173 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:00,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:01,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:01,747 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:11:02,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:03,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:04,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:05,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:06,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:06,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:07,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:08,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:09,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:10,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:11,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:11,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:12,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:13,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:14,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:15,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:11:15,171 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:11:15,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:16,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:16,307 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:16,751 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/config.yaml +2025-01-17 15:11:17,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:18,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:19,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:20,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:21,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:22,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:22,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:23,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:24,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:25,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:26,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:27,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:28,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:28,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:29,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:30,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:11:30,171 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:11:30,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:31,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:32,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:33,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:33,265 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:34,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:35,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:35,756 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:11:36,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:37,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:38,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:38,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:39,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:39,758 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:11:40,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:41,179 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:42,179 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:43,180 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:44,173 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:44,180 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:44,815 DEBUG SystemMonitor:2520816 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-17 15:11:44,816 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:11:45,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:11:45,171 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:11:45,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:46,181 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:47,181 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:48,181 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:49,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:49,245 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:50,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:51,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:52,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:53,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:54,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:55,173 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:55,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:56,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:57,183 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:58,183 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:59,183 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:00,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:12:00,171 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:12:00,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:00,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:01,183 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:02,183 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:03,183 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:04,183 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:05,017 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:12:05,017 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_98c68d418db5b73df1b0.mp4 with policy now +2025-01-17 15:12:05,017 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:12:05,018 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:12:05,019 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:12:05,020 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:12:05,184 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:05,291 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/z3xvgkha-media/videos/eval/0_eval_0_98c68d418db5b73df1b0.mp4 +2025-01-17 15:12:05,766 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:05,766 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/0_eval_0_98c68d418db5b73df1b0.mp4 +2025-01-17 15:12:05,766 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:12:05,766 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media +2025-01-17 15:12:05,766 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos +2025-01-17 15:12:05,766 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:12:06,095 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/ti2au1oi-media/videos/eval/0_eval_0_98c68d418db5b73df1b0.mp4 +2025-01-17 15:12:06,173 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:06,184 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:07,185 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:07,766 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:08,191 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:09,186 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:09,767 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:10,185 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:11,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:11,185 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:12,185 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:13,186 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:13,769 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:14,186 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:14,816 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:12:15,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:12:15,172 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:12:15,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:16,186 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:16,323 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:17,186 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:18,186 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:19,186 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:20,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:21,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:22,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:22,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:23,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:24,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:24,773 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:12:25,034 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/y26ewjg1-events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:12:25,188 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:25,773 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:26,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:27,188 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:27,729 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:27,774 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:28,188 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:29,188 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:29,774 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:30,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:12:30,172 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:12:30,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:31,188 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:31,775 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:32,188 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:33,188 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:33,609 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:33,776 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:34,188 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:35,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:35,777 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:36,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:37,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:38,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:39,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:39,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:40,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:41,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:42,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:43,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:44,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:44,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:44,817 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:12:45,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:12:45,172 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:12:45,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:46,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:47,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:47,781 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:48,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:49,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:49,271 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:50,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:51,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:52,191 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:53,191 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:54,191 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:54,421 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:55,191 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:55,784 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:56,192 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:56,641 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:12:56,641 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:12:56,642 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_fe99acb9f5fbe6b6c23e.mp4 with policy now +2025-01-17 15:12:56,642 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:12:56,642 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:12:56,642 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:12:56,784 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:12:56,784 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/1_eval_1_fe99acb9f5fbe6b6c23e.mp4 +2025-01-17 15:12:56,784 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:12:56,917 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/ai2sli5f-media/videos/eval/1_eval_1_fe99acb9f5fbe6b6c23e.mp4 +2025-01-17 15:12:57,193 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/fk6qtu11-media/videos/eval/1_eval_1_fe99acb9f5fbe6b6c23e.mp4 +2025-01-17 15:12:57,205 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:57,784 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:58,200 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:59,207 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:59,659 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:00,179 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:13:00,179 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:13:00,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:01,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:01,786 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:02,209 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:03,207 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:04,214 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:05,101 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:05,209 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:05,787 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:06,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:07,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:07,622 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:13:07,622 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:13:07,623 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:13:07,788 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:13:07,788 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:13:08,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:09,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:09,788 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:10,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:10,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:11,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:12,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:13,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:13,790 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:14,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:14,817 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:13:15,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:13:15,175 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:13:15,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:15,314 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:16,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:17,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:18,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:19,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:20,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:21,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:21,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:22,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:23,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:24,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:25,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:26,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:26,689 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:27,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:27,795 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:28,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:29,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:30,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:13:30,175 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:13:30,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:31,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:32,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:32,298 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:33,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:34,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:35,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:36,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:37,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:38,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:38,253 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:39,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:39,347 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:13:39,347 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a9b669c92c8beee6656c.mp4 with policy now +2025-01-17 15:13:39,351 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:13:39,352 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:13:39,352 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:13:39,352 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:13:39,609 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/m6x7gw4q-media/videos/eval/2_eval_2_a9b669c92c8beee6656c.mp4 +2025-01-17 15:13:39,799 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:39,799 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:13:39,799 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/2_eval_2_a9b669c92c8beee6656c.mp4 +2025-01-17 15:13:39,799 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:13:40,101 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/gfad54gh-media/videos/eval/2_eval_2_a9b669c92c8beee6656c.mp4 +2025-01-17 15:13:40,220 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:41,218 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:41,799 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:42,218 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:43,220 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:43,800 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:44,065 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:44,228 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:44,818 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:13:45,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:13:45,182 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:13:45,229 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:45,801 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:46,219 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:47,220 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:47,801 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:48,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:48,802 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:13:49,100 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/0g95dqdn-events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:13:49,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:49,450 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:13:49,450 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:13:49,450 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:49,450 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:13:49,802 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:49,802 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:13:50,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:51,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:51,803 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:52,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:53,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:53,804 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:54,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:54,919 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:55,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:56,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:57,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:57,805 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:58,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:59,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:00,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:14:00,176 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:14:00,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:00,270 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:01,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:02,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:03,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:04,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:05,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:05,651 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:05,808 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:06,223 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:07,224 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:07,809 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:08,224 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:09,224 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:09,809 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:10,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:11,180 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:11,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:11,810 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:12,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:13,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:14,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:14,818 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:14:15,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:14:15,176 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:14:15,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:16,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:16,331 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:17,226 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:18,226 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:19,226 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:20,232 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:21,116 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:14:21,116 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_2988eff819b475f13199.mp4 with policy now +2025-01-17 15:14:21,118 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:14:21,119 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:14:21,119 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:14:21,120 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:14:21,226 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:21,388 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/byrjrb8o-media/videos/eval/3_eval_3_2988eff819b475f13199.mp4 +2025-01-17 15:14:21,813 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:21,814 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:14:21,814 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/3_eval_3_2988eff819b475f13199.mp4 +2025-01-17 15:14:21,814 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:14:22,113 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/b2bazao1-media/videos/eval/3_eval_3_2988eff819b475f13199.mp4 +2025-01-17 15:14:22,155 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:22,248 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:23,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:23,814 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:24,242 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:25,252 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:25,815 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:26,244 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:27,193 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:27,245 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:28,242 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:29,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:29,816 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:30,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:14:30,190 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:14:30,245 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:30,502 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:14:30,502 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:14:30,502 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:14:30,817 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:14:30,817 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:14:31,261 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:31,817 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:32,244 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:32,261 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:33,248 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:33,818 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:34,251 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:35,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:35,819 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:36,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:37,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:38,184 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:38,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:39,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:40,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:41,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:42,244 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:43,244 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:44,184 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:44,244 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:44,818 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:14:45,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:14:45,177 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:14:45,244 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:46,244 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:47,244 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:47,823 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:48,245 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:49,245 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:49,946 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:50,245 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:51,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:51,824 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:52,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:53,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:53,825 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:54,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:54,946 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:55,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:55,826 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:56,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:57,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:57,827 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:58,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:59,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:59,827 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:00,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:15:00,178 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:15:00,247 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:00,295 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:01,247 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:01,828 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:02,248 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:02,504 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:15:02,504 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_8487a970815421048be9.mp4 with policy now +2025-01-17 15:15:02,509 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:15:02,509 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:15:02,510 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:15:02,510 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:15:02,773 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/avmj8hjn-media/videos/eval/4_eval_4_8487a970815421048be9.mp4 +2025-01-17 15:15:02,828 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:15:02,829 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/4_eval_4_8487a970815421048be9.mp4 +2025-01-17 15:15:02,829 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:15:03,116 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/sd8v7i4r-media/videos/eval/4_eval_4_8487a970815421048be9.mp4 +2025-01-17 15:15:03,253 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:03,829 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:04,255 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:05,261 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:06,196 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:06,272 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:07,256 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:07,830 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:08,276 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:09,256 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:09,831 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:10,284 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:11,204 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:11,276 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:11,832 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:12,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:13,280 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:14,284 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:14,819 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:15:14,833 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:15:15,138 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/xyaht6es-events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:15:15,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:15:15,179 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:15:15,285 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:15,556 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:15:15,556 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:15:15,556 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:15:15,834 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:15,834 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:15:16,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:16,781 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:17,301 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:17,835 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:18,302 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:19,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:19,836 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:20,295 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:21,302 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:21,877 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:22,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:23,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:23,837 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:24,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:25,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:26,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:27,198 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:27,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:28,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:29,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:30,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:15:30,177 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:15:30,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:31,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:32,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:32,297 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:33,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:34,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:35,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:36,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:37,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:38,198 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:38,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:39,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:40,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:41,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:42,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:43,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:44,198 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:44,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:44,819 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:15:45,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:15:45,177 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:15:45,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:46,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:47,295 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:48,295 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:49,316 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:49,472 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:49,846 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:50,018 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:15:50,018 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_5eccb1be08111ca14e10.mp4 with policy now +2025-01-17 15:15:50,020 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:15:50,021 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:15:50,021 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:15:50,021 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:15:50,308 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:50,319 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/ornukn1e-media/videos/eval/5_eval_5_5eccb1be08111ca14e10.mp4 +2025-01-17 15:15:50,847 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:15:50,847 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/5_eval_5_5eccb1be08111ca14e10.mp4 +2025-01-17 15:15:50,847 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:15:51,148 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/1jblr2x5-media/videos/eval/5_eval_5_5eccb1be08111ca14e10.mp4 +2025-01-17 15:15:51,310 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:51,847 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:52,303 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:53,312 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:54,310 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:54,745 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:55,317 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:55,848 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:56,309 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:57,313 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:57,849 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:58,309 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:58,849 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:15:59,312 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:59,608 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:15:59,609 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:15:59,609 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:15:59,850 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:15:59,850 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:59,850 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:15:59,905 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:00,180 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:16:00,180 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:16:00,316 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:01,316 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:01,850 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:02,330 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:03,320 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:03,851 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:04,325 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:05,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:05,322 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:05,852 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:06,322 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:07,322 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:08,322 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:09,322 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:10,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:10,324 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:11,324 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:12,324 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:13,324 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:14,324 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:14,820 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:16:15,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:16:15,178 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:16:15,260 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:15,324 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:16,325 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:17,325 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:18,325 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:19,325 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:20,325 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:21,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:21,326 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:22,326 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:23,326 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:24,326 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:25,326 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:26,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:26,326 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:27,326 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:28,326 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:29,327 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:30,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:16:30,178 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:16:30,327 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:31,284 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:31,327 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:32,338 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:33,049 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:16:33,049 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_5f3fdb589a9c504e4946.mp4 with policy now +2025-01-17 15:16:33,051 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:16:33,052 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:16:33,052 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:16:33,052 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:16:33,326 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/sdomd3mu-media/videos/eval/6_eval_6_5f3fdb589a9c504e4946.mp4 +2025-01-17 15:16:33,333 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:33,862 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:33,862 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:16:33,862 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/6_eval_6_5f3fdb589a9c504e4946.mp4 +2025-01-17 15:16:33,862 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:16:34,177 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/uaptubbe-media/videos/eval/6_eval_6_5f3fdb589a9c504e4946.mp4 +2025-01-17 15:16:34,337 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:35,335 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:35,863 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:36,359 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:36,720 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:37,344 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:37,863 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:38,349 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:39,368 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:40,383 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:41,356 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:41,865 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:42,062 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:42,356 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:42,660 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:16:42,660 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:16:42,661 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:16:42,865 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:16:42,865 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:16:43,155 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/6uc5wvic-events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:16:43,361 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:43,866 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:44,367 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:44,820 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:16:45,181 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:16:45,182 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:16:45,361 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:45,866 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:46,353 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:47,353 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:47,526 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:47,867 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:48,353 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:49,353 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:50,353 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:51,353 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:52,353 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:53,113 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:53,353 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:53,869 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:54,353 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:55,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:56,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:57,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:58,217 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:58,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:59,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:00,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:17:00,178 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:17:00,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:01,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:02,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:03,262 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:03,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:04,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:05,355 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:06,355 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:07,355 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:08,355 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:09,218 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:09,355 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:10,355 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:11,356 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:12,356 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:13,356 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:13,878 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:17:14,370 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:14,397 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:14,511 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:17:14,511 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_eab2215188c302c460ba.mp4 with policy now +2025-01-17 15:17:14,514 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:17:14,522 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:17:14,522 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:17:14,522 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:17:14,791 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/j00t70i9-media/videos/eval/7_eval_7_eab2215188c302c460ba.mp4 +2025-01-17 15:17:14,821 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:17:14,878 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:17:14,879 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/7_eval_7_eab2215188c302c460ba.mp4 +2025-01-17 15:17:14,879 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:17:15,173 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/7bncykqv-media/videos/eval/7_eval_7_eab2215188c302c460ba.mp4 +2025-01-17 15:17:15,195 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:17:15,196 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:17:15,398 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:15,879 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:17:16,382 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:17,397 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:18,376 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:19,407 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:19,880 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:17:20,226 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:20,396 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:21,376 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:22,397 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:23,398 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:23,882 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:17:24,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:24,712 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:17:24,712 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:17:24,713 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:17:24,882 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:17:24,882 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:17:25,236 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:25,417 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:25,882 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:17:26,414 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:27,415 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:27,883 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:17:28,433 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:29,403 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:29,884 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:17:30,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:17:30,178 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:17:30,268 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:30,403 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:31,403 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:32,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:33,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:34,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:35,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:36,230 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:36,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:37,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:38,405 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:39,405 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:40,405 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:41,231 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:41,405 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:42,405 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:43,405 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:44,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:44,821 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:17:45,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:17:45,179 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:17:45,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:46,302 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:46,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:47,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:48,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:49,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:50,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:51,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:52,231 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:52,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:53,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:54,407 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:55,437 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:56,446 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:56,692 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:17:56,692 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_50d0e7e81a609c529244.mp4 with policy now +2025-01-17 15:17:56,696 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:17:56,696 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:17:56,696 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:17:56,697 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:17:56,894 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:17:56,894 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/8_eval_8_50d0e7e81a609c529244.mp4 +2025-01-17 15:17:56,895 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:17:56,958 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/4j68napr-media/videos/eval/8_eval_8_50d0e7e81a609c529244.mp4 +2025-01-17 15:17:57,254 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/pezr5psf-media/videos/eval/8_eval_8_50d0e7e81a609c529244.mp4 +2025-01-17 15:17:57,442 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:57,754 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:57,895 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:17:58,449 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:59,446 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:59,895 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:00,206 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:18:00,206 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:18:00,445 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:01,453 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:01,896 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:02,470 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:03,427 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:03,464 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:04,459 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:05,462 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:05,898 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:06,480 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:07,463 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:08,478 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:08,766 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:18:08,766 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:18:08,766 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:08,766 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:18:08,899 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:18:08,899 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:18:09,171 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/8ey6st4m-events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:18:09,466 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:09,899 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:10,470 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:11,470 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:12,475 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:13,478 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:13,901 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:14,305 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:14,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:14,822 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:18:15,214 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:18:15,214 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:18:15,501 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:16,496 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:17,491 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:18,487 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:19,487 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:19,903 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:20,011 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:20,487 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:21,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:22,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:23,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:24,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:25,248 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:25,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:26,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:27,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:27,906 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:28,489 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:29,489 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:30,188 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:18:30,188 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:18:30,336 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:30,489 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:31,489 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:32,489 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:33,489 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:33,908 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:34,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:35,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:36,248 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:36,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:37,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:38,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:39,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:40,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:41,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:42,248 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:42,491 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:43,491 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:44,491 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:44,822 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:18:45,204 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:18:45,204 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:18:45,498 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:45,913 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:45,913 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/9_eval_9_9a2233becb7986a47e63.mp4 +2025-01-17 15:18:45,913 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:18:45,926 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:18:45,926 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_9a2233becb7986a47e63.mp4 with policy now +2025-01-17 15:18:45,930 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:18:45,931 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:18:45,931 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:18:45,931 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:18:46,179 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/bo3mrt93-media/videos/eval/9_eval_9_9a2233becb7986a47e63.mp4 +2025-01-17 15:18:46,471 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/utzcmyyr-media/videos/eval/9_eval_9_9a2233becb7986a47e63.mp4 +2025-01-17 15:18:46,529 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:46,913 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:18:47,513 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:47,914 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:47,987 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:48,520 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:49,525 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:50,526 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:51,527 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:51,915 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:52,525 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:53,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:53,657 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:53,916 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:54,533 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:55,532 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:56,530 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:56,824 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:18:56,824 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:18:56,825 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:18:56,917 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:18:56,917 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:18:57,537 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:57,918 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:58,563 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:59,537 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:59,538 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:59,918 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:00,216 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:19:00,216 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:19:00,533 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:01,535 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:01,919 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:02,545 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:03,541 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:04,561 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:05,268 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:05,557 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:06,539 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:07,539 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:07,921 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:08,539 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:09,539 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:10,539 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:11,266 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:11,539 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:12,539 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:13,540 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:14,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:14,823 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:19:15,195 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:19:15,195 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:19:15,541 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:16,334 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:16,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:17,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:18,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:19,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:20,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:21,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:22,266 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:22,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:23,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:24,543 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:25,543 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:26,543 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:27,267 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:27,543 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:28,543 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:29,543 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:30,196 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:19:30,196 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:19:30,543 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:31,544 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:32,348 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:32,544 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:33,562 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:33,934 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:34,553 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:34,637 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:19:34,638 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_dc26db5ffbb288ebf69b.mp4 with policy now +2025-01-17 15:19:34,681 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:19:34,681 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:19:34,682 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:19:34,682 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:19:34,924 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/g9mnvilh-media/videos/eval/10_eval_10_dc26db5ffbb288ebf69b.mp4 +2025-01-17 15:19:34,935 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:19:34,935 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/10_eval_10_dc26db5ffbb288ebf69b.mp4 +2025-01-17 15:19:34,935 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:19:35,227 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/02glaz0y-media/videos/eval/10_eval_10_dc26db5ffbb288ebf69b.mp4 +2025-01-17 15:19:35,562 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:35,935 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:36,559 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:37,584 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:38,272 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:38,576 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:39,569 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:39,937 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:40,594 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:41,569 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:42,588 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:43,529 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:43,592 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:43,938 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:44,590 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:44,823 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:19:44,885 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:19:44,885 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:19:44,885 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:19:44,939 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:19:44,939 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:19:45,201 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:19:45,201 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:19:45,243 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/a7pqmqj4-events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:19:45,624 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:45,939 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:46,620 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:47,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:47,940 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:48,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:48,913 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:49,611 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:49,940 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:50,616 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:51,617 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:52,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:53,628 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:54,271 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:54,623 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:55,632 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:56,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:57,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:57,944 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:58,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:59,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:59,818 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:00,196 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:20:00,196 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:20:00,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:01,616 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:02,616 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:03,616 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:04,616 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:05,274 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:05,617 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:06,617 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:07,617 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:08,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:09,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:10,274 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:10,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:11,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:12,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:13,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:14,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:14,823 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:20:15,196 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:20:15,197 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:20:15,283 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:15,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:16,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:17,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:17,952 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:20:18,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:19,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:20,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:21,275 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:21,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:22,620 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:23,620 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:24,629 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:24,955 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/11_eval_11_f83167404e8bb0859778.mp4 +2025-01-17 15:20:24,955 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:20:24,976 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:20:24,976 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_f83167404e8bb0859778.mp4 with policy now +2025-01-17 15:20:24,986 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:20:24,987 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:20:24,987 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:20:24,987 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:20:25,227 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/sm0rehjx-media/videos/eval/11_eval_11_f83167404e8bb0859778.mp4 +2025-01-17 15:20:25,554 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/gbin574l-media/videos/eval/11_eval_11_f83167404e8bb0859778.mp4 +2025-01-17 15:20:25,660 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:25,955 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:20:25,955 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:20:26,643 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:27,275 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:27,661 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:27,956 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:20:28,665 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:29,665 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:29,957 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:20:30,231 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:20:30,232 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:20:30,667 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:31,664 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:32,681 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:32,817 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:33,695 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:33,958 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:20:34,688 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:35,678 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:36,685 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:37,694 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:38,284 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:38,705 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:39,697 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:40,712 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:41,716 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:42,740 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:43,306 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:43,728 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:44,747 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:44,824 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:20:45,226 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:20:45,226 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:20:45,751 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:46,746 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:47,739 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:48,327 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:48,739 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:49,742 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:50,781 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:51,772 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:52,749 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:53,747 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:54,318 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:54,774 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:55,778 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:56,787 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:57,795 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:58,792 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:59,790 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:00,217 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:21:00,217 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:21:00,376 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:00,805 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:01,802 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:02,809 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:03,819 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:04,833 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:05,818 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:06,350 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:06,842 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:07,817 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:08,831 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:09,833 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:10,832 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:11,856 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:12,332 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:12,857 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:13,873 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:14,824 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:21:14,854 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:15,223 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:21:15,223 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:21:15,851 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:16,854 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:17,332 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:17,859 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:18,872 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:19,877 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:20,894 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:21,890 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:22,362 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:22,875 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:23,890 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:24,909 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:25,879 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:26,907 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:27,884 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:28,342 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:28,882 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:29,891 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:30,227 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:21:30,227 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:21:30,883 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:31,903 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:32,892 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:33,362 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:33,890 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:34,902 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:35,887 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:36,893 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:37,900 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:38,915 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:39,344 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:39,918 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:40,910 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:41,924 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:42,921 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:43,933 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:44,350 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:44,825 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:21:44,946 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:45,230 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:21:45,230 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:21:45,945 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:46,949 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:47,954 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:48,949 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:49,363 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:49,958 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:50,953 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:51,988 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:52,961 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:53,958 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:54,957 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:55,348 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:55,975 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:56,971 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:57,964 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:58,979 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:59,987 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:00,247 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:22:00,247 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:22:00,358 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:01,009 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:02,001 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:02,985 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:04,004 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:04,997 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:05,359 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:06,020 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:07,007 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:08,027 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:09,016 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:10,013 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:10,364 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:11,026 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:12,036 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:13,024 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:14,040 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:14,825 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:22:15,050 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:15,247 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:22:15,247 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:22:15,366 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:16,028 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:17,045 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:18,048 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:19,062 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:20,051 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:20,383 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:21,063 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:22,068 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:23,065 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:24,071 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:25,094 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:25,388 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:26,109 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:27,078 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:28,086 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:29,088 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:30,088 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:30,229 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:22:30,229 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:22:30,413 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:31,106 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:32,119 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:33,126 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:34,127 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:35,114 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:36,142 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:36,403 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:37,107 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:38,114 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:39,121 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:40,117 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:41,149 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:42,145 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:42,398 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:43,121 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:44,147 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:44,826 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:22:45,149 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:45,250 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:22:45,251 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:22:46,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:47,138 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:47,410 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:48,169 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:49,145 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:50,167 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:51,168 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:52,170 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:52,417 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:53,205 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:54,179 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:55,185 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:56,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:57,204 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:58,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:58,414 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:59,237 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:00,218 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:00,262 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:23:00,262 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:23:01,238 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:02,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:03,223 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:03,433 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:04,238 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:05,229 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:06,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:07,237 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:08,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:08,437 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:09,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:10,249 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:11,257 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:12,257 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:13,263 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:14,263 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:14,417 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:14,826 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:23:15,260 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:23:15,260 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:23:15,305 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:16,265 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:17,266 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:18,275 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:19,265 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:19,430 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:20,280 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:21,265 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:22,304 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:23,283 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:24,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:25,282 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:25,427 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:26,302 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:27,336 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:28,312 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:29,303 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:30,247 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:23:30,247 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:23:30,300 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:30,436 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:31,336 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:32,338 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:33,319 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:34,346 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:35,330 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:36,337 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:36,433 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:37,333 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:38,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:39,369 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:40,357 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:41,335 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:41,434 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:42,349 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:43,362 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:44,358 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:44,827 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:23:45,264 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:23:45,265 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:23:45,375 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:46,395 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:46,436 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:47,346 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:48,366 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:49,359 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:50,362 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:51,382 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:51,447 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:52,362 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:53,378 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:54,382 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:55,380 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:56,414 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:56,452 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:57,384 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:58,385 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:59,391 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:00,278 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:24:00,278 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:24:00,385 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:01,422 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:02,415 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:02,449 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:03,421 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:04,447 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:05,450 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:06,444 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:07,444 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:07,474 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:08,447 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:09,466 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:10,452 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:11,453 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:12,459 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:13,469 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:13,471 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:14,469 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:14,827 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:24:15,264 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:24:15,264 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:24:15,486 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:16,503 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:17,509 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:18,471 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:18,481 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:19,491 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:20,509 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:21,479 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:22,483 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:23,473 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:23,510 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:24,515 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:25,500 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:26,520 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:27,507 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:28,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:28,528 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:29,531 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:30,256 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:24:30,256 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:24:30,565 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:31,550 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:32,551 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:33,556 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:34,478 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:34,594 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:35,585 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:36,583 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:37,605 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:38,613 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:39,506 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:39,603 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:40,662 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:41,617 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:42,627 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:43,628 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:44,638 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:44,828 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:24:44,828 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:45,264 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:24:45,264 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:24:45,644 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:46,643 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:47,647 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:48,647 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:49,642 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:50,478 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:50,642 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:51,644 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:52,674 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:53,675 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:54,675 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:55,487 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:55,672 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:56,700 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:57,664 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:58,668 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:59,684 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:00,289 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:25:00,289 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:25:00,494 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:00,675 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:01,669 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:02,701 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:03,679 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:04,704 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:05,495 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:05,685 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:06,696 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:07,754 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:08,712 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:09,701 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:10,508 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:10,709 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:11,712 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:12,702 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:13,689 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:14,695 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:14,828 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:25:15,266 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:25:15,266 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:25:15,527 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:15,702 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:16,700 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:17,720 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:18,732 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:19,724 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:20,731 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:21,524 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:21,729 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:22,743 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:23,737 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:24,754 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:25,777 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:26,529 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:26,742 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:27,762 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:28,757 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:29,745 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:30,302 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:25:30,302 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:25:30,772 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:31,755 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:32,508 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:32,770 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:33,780 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:34,751 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:35,793 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:36,806 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:37,526 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:37,798 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:38,803 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:39,797 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:40,797 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:41,800 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:42,527 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:42,805 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:43,803 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:44,826 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:44,829 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:25:45,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:25:45,294 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:25:45,803 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:46,809 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:47,534 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:47,806 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:48,820 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:49,828 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:50,840 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:51,831 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:52,824 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:53,533 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:53,828 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:54,852 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:55,851 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:56,859 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:57,848 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:58,850 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:59,532 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:59,852 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:00,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:26:00,293 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:26:00,898 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:01,852 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:02,879 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:03,875 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:04,547 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:04,879 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:05,881 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:06,897 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:07,882 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:08,900 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:09,562 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:09,918 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:10,910 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:11,902 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:12,918 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:13,903 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:14,585 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:14,829 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:26:14,911 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:15,309 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:26:15,310 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:26:15,914 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:16,913 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:17,956 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:18,922 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:19,608 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:19,945 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:20,948 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:21,958 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:22,948 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:23,946 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:24,982 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:25,570 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:25,981 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:26,968 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:27,969 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:28,970 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:29,967 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:30,310 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:26:30,310 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:26:30,960 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:31,568 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:31,959 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:32,966 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:33,986 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:35,025 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:35,988 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:36,592 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:37,004 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:37,980 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:38,987 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:39,991 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:41,022 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:41,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:42,013 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:43,024 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:44,005 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:44,829 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:26:45,017 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:45,320 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:26:45,320 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:26:46,017 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:47,011 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:47,581 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:48,016 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:49,040 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:50,016 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:51,022 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:52,034 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:52,603 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:53,031 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:54,057 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:55,045 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:56,039 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:57,047 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:58,050 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:58,573 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:59,058 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:00,051 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:00,298 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:27:00,298 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:27:01,059 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:02,068 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:03,059 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:03,605 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:04,068 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:05,097 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:06,084 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:07,080 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:08,081 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:09,095 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:09,585 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:10,078 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:11,077 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:12,094 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:13,100 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:14,101 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:14,600 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:14,830 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:27:15,121 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:15,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:27:15,293 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:27:16,141 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:17,125 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:18,143 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:19,156 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:20,166 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:20,595 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:21,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:22,157 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:23,157 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:24,148 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:25,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:26,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:26,587 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:27,202 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:28,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:29,197 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:30,199 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:30,323 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:27:30,323 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:27:31,230 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:31,609 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:32,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:33,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:34,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:35,238 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:36,247 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:36,611 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:37,275 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:38,254 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:39,255 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:40,265 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:41,256 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:41,634 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:42,255 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:43,242 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:44,267 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:44,830 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:27:45,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:45,338 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:27:45,339 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:27:46,295 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:46,669 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:47,263 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:48,270 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:49,267 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:50,281 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:51,268 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:52,269 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:52,636 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:53,268 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:54,290 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:55,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:56,315 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:57,331 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:58,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:58,633 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:59,327 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:00,339 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:28:00,339 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:28:00,381 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:01,360 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:02,357 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:03,379 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:03,656 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:04,351 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:05,364 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:06,363 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:07,381 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:08,383 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:09,370 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:09,652 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:10,387 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:11,416 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:12,391 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:13,378 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:14,383 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:14,657 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:14,831 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:28:15,376 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:28:15,376 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:28:15,417 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:16,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:17,414 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:18,423 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:19,417 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:19,674 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:20,433 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:21,416 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:22,417 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:23,444 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:24,427 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:24,675 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:25,440 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:26,475 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:27,443 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:28,479 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:29,469 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:29,681 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:30,389 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:28:30,389 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:28:30,465 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:31,455 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:32,470 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:33,474 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:34,487 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:34,711 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:35,484 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:36,486 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:37,497 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:38,504 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:39,519 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:39,713 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:40,496 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:41,526 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:42,508 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:43,549 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:44,516 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:44,831 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:28:44,832 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:45,366 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:28:45,366 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:28:45,536 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:46,526 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:47,540 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:48,518 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:49,522 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:50,547 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:50,709 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:51,538 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:52,585 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:53,546 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:54,576 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:55,557 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:55,709 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:56,570 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:57,545 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:58,566 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:59,563 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:00,367 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:29:00,367 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:29:00,576 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:00,760 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:01,577 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:02,572 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:03,567 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:04,563 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:05,570 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:05,760 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:06,575 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:07,568 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:08,576 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:09,593 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:10,601 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:10,762 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:11,606 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:12,601 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:13,601 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:14,645 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:14,831 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:29:15,391 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:29:15,392 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:29:15,613 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:15,778 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:16,639 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:17,654 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:18,630 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:19,646 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:20,649 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:20,779 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:21,663 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:22,664 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:23,662 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:24,684 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:25,691 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:26,657 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:26,770 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:27,666 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:28,671 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:29,689 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:30,386 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:29:30,387 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:29:30,679 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:31,668 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:31,791 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:32,695 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:33,717 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:34,669 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:35,685 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:36,687 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:37,739 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:37,777 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:38,676 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:39,694 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:40,703 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:41,680 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:42,691 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:42,778 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:43,677 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:44,685 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:44,832 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:29:45,368 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:29:45,368 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:29:45,703 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:46,714 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:47,710 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:48,713 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:48,774 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:49,728 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:50,723 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:51,735 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:52,735 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:53,736 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:53,797 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:54,766 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:55,762 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:56,756 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:57,760 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:58,781 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:59,787 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:59,793 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:00,390 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:30:00,391 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:30:00,791 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:01,786 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:02,794 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:03,788 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:04,808 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:04,827 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:05,818 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:06,793 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:07,813 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:08,832 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:09,816 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:09,849 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:10,856 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:11,833 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:12,849 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:13,835 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:14,832 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:30:14,842 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:15,377 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:30:15,378 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:30:15,454 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:15,847 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:16,865 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:17,831 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:18,841 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:19,850 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:20,839 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:20,855 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:21,846 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:22,870 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:23,857 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:24,856 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:25,840 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:25,888 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:26,899 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:27,882 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:28,906 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:29,901 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:30,395 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:30:30,395 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:30:30,842 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:30,907 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:31,927 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:32,902 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:33,911 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:34,912 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:35,918 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:36,840 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:36,929 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:37,901 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:38,938 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:39,911 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:40,922 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:41,879 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:41,938 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:42,925 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:43,924 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:44,833 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:30:44,949 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:45,395 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:30:45,395 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:30:45,954 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:46,955 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:47,846 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:47,978 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:48,969 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:49,978 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:50,983 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:51,990 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:52,855 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:53,023 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:54,022 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:55,009 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:56,025 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:57,044 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:58,027 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:58,852 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:59,036 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:00,036 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:00,387 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:31:00,388 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:31:01,065 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:02,064 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:03,039 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:03,881 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:04,074 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:05,070 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:06,066 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:07,039 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:08,059 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:09,074 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:09,877 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:10,048 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:11,073 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:12,074 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:13,084 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:14,097 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:14,833 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:31:15,093 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:15,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:31:15,405 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:31:15,568 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:16,124 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:17,081 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:18,097 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:19,092 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:20,096 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:20,869 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:21,122 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:22,092 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:23,095 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:24,125 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:25,098 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:25,869 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:26,140 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:27,095 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:28,099 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:29,109 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:30,118 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:30,399 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:31:30,399 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:31:30,908 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:31,107 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:32,124 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:33,117 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:34,123 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:35,166 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:36,145 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:36,892 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:37,138 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:38,134 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:39,162 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:40,148 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:41,156 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:41,901 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:42,157 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:43,147 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:44,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:44,834 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:31:45,151 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:45,412 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:31:45,413 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:31:46,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:47,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:47,890 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:48,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:49,198 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:50,207 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:51,185 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:52,194 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:53,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:53,877 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:54,197 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:55,194 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:56,204 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:57,226 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:58,215 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:58,886 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:59,194 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:00,257 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:00,425 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:32:00,425 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:32:01,233 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:02,227 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:03,234 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:04,239 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:04,885 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:05,254 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:06,256 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:07,261 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:08,262 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:09,250 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:09,910 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:10,257 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:11,276 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:12,280 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:13,317 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:14,330 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:14,834 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:32:15,282 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:15,418 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:32:15,418 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:32:15,507 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:16,284 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:17,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:18,313 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:19,296 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:20,319 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:20,926 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:21,305 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:22,314 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:23,333 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:24,311 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:25,306 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:26,327 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:26,912 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:27,342 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:28,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:29,350 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:30,346 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:30,453 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:32:30,453 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:32:31,345 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:31,942 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:32,369 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:33,373 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:34,340 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:35,363 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:36,349 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:37,401 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:37,918 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:38,366 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:39,376 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:40,397 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:41,373 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:42,371 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:42,932 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:43,430 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:44,410 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:44,834 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:32:45,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:45,429 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:32:45,429 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:32:46,385 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:47,394 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:48,429 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:48,931 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:49,427 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:50,429 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:51,449 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:52,448 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:53,435 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:54,489 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:54,922 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:55,451 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:56,455 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:57,479 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:58,478 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:59,474 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:59,936 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:00,445 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:33:00,445 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:33:00,489 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:01,463 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:02,474 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:03,505 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:04,514 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:04,953 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:05,497 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:06,503 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:07,507 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:08,520 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:09,504 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:10,510 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:10,943 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:11,506 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:12,508 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:13,537 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:14,550 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:14,835 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:33:15,466 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:33:15,466 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:33:15,532 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:16,533 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:16,931 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:17,557 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:18,538 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:19,543 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:20,536 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:21,560 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:21,939 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:22,563 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:23,561 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:24,587 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:25,573 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:26,568 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:27,571 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:27,937 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:28,573 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:29,581 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:30,436 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:33:30,436 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:33:30,611 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:31,602 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:32,584 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:33,014 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:33,592 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:34,593 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:35,604 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:36,592 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:37,596 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:38,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:38,963 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:39,611 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:40,591 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:41,621 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:42,616 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:43,601 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:44,629 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:44,835 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:33:44,836 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:45,450 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:33:45,450 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:33:45,621 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:46,620 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:47,654 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:48,636 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:49,635 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:49,957 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:50,661 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:51,652 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:52,672 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:53,649 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:54,661 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:55,675 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:55,952 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:56,672 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:57,667 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:58,668 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:59,677 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:00,433 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:34:00,434 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:34:00,669 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:00,960 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:01,685 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:02,709 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:03,698 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:04,703 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:05,288 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:34:05,694 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:05,963 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:06,732 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:07,726 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:08,737 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:09,742 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:10,763 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:10,963 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:11,766 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:12,824 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:13,855 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:14,794 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:14,836 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:34:15,520 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:34:15,521 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:34:15,782 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:16,024 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:16,838 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:17,799 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:18,832 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:19,828 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:20,865 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:21,024 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:21,848 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:22,850 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:23,830 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:24,950 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:25,871 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:26,057 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:26,921 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:27,878 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:29,004 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:29,967 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:30,516 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:34:30,516 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:34:30,901 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:31,958 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:32,021 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:33,000 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:33,995 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:34,915 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:35,964 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:36,991 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:37,066 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:37,978 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:39,066 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:39,998 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:41,124 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:42,047 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:42,131 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:43,165 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:44,084 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:44,836 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:34:45,089 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:45,495 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:34:45,495 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:34:46,103 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:47,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:48,105 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:48,115 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:49,148 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:50,196 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:51,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:52,130 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:53,110 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:53,131 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:54,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:55,135 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:56,202 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:57,217 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:58,113 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:58,194 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:59,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:00,240 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:00,568 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:35:00,568 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:35:01,286 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:02,262 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:03,262 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:03,330 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:04,318 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:05,360 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:06,326 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:07,321 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:08,328 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:09,079 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:09,361 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:10,301 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:11,351 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:12,389 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:13,333 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:14,135 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:14,529 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:14,837 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:35:15,393 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:15,500 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:35:15,500 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:35:16,510 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:17,440 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:18,415 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:19,462 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:20,112 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:20,479 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:21,578 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:22,465 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:23,492 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:24,491 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:25,173 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:25,570 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:26,572 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:27,518 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:28,504 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:29,503 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:30,247 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:30,481 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:35:30,481 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:35:30,536 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:31,529 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:32,545 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:33,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:34,568 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:35,544 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:36,108 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:36,529 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:37,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:38,584 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:39,661 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:40,571 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:41,115 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:41,617 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:42,714 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:43,633 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:44,691 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:44,837 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:35:45,481 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:35:45,481 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:35:45,647 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:46,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:46,640 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:47,643 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:48,666 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:49,651 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:50,713 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:51,676 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:52,195 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:52,685 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:53,709 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:54,737 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:55,730 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:56,783 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:57,198 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:57,797 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:58,729 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:59,823 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:00,464 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:36:00,465 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:36:00,786 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:01,861 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:02,765 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:03,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:03,777 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:04,800 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:05,857 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:06,818 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:08,049 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:08,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:08,817 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:09,856 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:10,837 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:11,914 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:12,878 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:13,223 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:13,950 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:14,838 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:36:14,953 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:15,550 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:36:15,550 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:36:15,967 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:16,958 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:17,997 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:18,224 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:18,932 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:20,035 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:20,989 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:22,032 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:22,997 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:23,235 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:23,988 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:24,992 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:25,975 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:27,039 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:28,054 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:28,343 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:29,065 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:30,059 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:30,617 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:36:30,617 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:36:31,054 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:32,115 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:33,168 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:34,130 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:34,298 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:35,135 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:36,106 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:37,196 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:38,164 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:39,132 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:40,130 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:40,267 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:41,132 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:42,116 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:43,278 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:44,195 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:44,846 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:36:45,264 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:45,597 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:36:45,597 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:36:45,731 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:46,226 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:47,219 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:48,205 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:49,282 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:50,298 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:51,244 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:51,281 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:52,264 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:53,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:54,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:55,232 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:56,262 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:56,329 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:57,273 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:58,281 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:59,345 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:00,345 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:00,578 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:37:00,579 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:37:01,348 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:01,460 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:02,431 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:03,466 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:04,337 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:05,343 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:06,427 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:07,394 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:07,417 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:08,399 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:09,409 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:10,477 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:11,371 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:12,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:12,414 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:13,433 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:14,451 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:14,838 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:37:15,424 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:15,579 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:37:15,579 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:37:16,460 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:17,504 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:18,367 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:18,497 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:19,440 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:20,483 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:21,463 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:22,525 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:23,463 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:24,330 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:24,585 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:25,495 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:26,578 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:27,511 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:28,479 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:29,492 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:30,305 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:30,537 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:30,605 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:37:30,605 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:37:31,512 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:32,520 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:33,636 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:34,572 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:35,388 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:35,603 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:36,584 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:37,592 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:38,577 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:39,568 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:40,414 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:40,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:41,657 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:42,633 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:43,663 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:44,662 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:44,839 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:37:45,693 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:45,732 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:37:45,732 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:37:45,845 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:46,670 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:47,834 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:48,762 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:49,668 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:50,713 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:51,442 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:51,731 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:52,792 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:53,700 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:54,702 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:55,720 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:56,769 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:57,430 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:57,810 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:58,752 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:59,791 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:00,595 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:38:00,595 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:38:00,753 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:01,840 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:02,451 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:02,813 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:03,816 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:04,815 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:05,847 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:06,826 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:07,492 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:07,889 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:08,846 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:09,858 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:10,878 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:11,844 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:12,962 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:13,457 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:13,946 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:14,839 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:38:14,941 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:15,651 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:38:15,651 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:38:15,937 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:17,003 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:18,036 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:19,013 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:19,441 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:19,952 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:20,957 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:22,017 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:23,079 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:24,063 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:24,501 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:25,058 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:26,018 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:27,041 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:28,078 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:29,042 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:30,059 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:30,494 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:30,625 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:38:30,626 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:38:31,140 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:32,106 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:33,135 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:34,140 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:35,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:35,497 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:36,166 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:37,162 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:38,167 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:39,191 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:40,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:40,617 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:41,232 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:42,198 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:43,198 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:44,269 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:44,840 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:38:45,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:45,732 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:38:45,732 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:38:45,816 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:46,263 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:47,249 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:48,276 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:49,344 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:50,303 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:51,307 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:51,546 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:52,343 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:53,344 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:54,414 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:55,389 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:56,448 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:56,561 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:57,394 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:58,461 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:59,427 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:00,446 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:00,689 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:39:00,689 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:39:01,517 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:01,589 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:02,517 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:03,495 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:04,498 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:05,560 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:06,543 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:06,590 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:07,555 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:08,510 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:09,519 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:10,500 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:11,583 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:11,599 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:12,506 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:13,516 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:14,554 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:14,840 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:39:15,604 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:15,608 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:39:15,608 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:39:16,644 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:16,671 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:17,550 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:18,552 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:19,651 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:20,683 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:21,622 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:22,631 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:22,643 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:23,678 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:24,718 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:25,680 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:26,732 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:27,744 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:28,606 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:28,673 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:29,704 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:30,653 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:39:30,654 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:39:30,757 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:31,668 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:32,723 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:33,639 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:33,694 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:34,757 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:35,824 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:36,767 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:37,801 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:38,718 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:39,591 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:39,761 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:40,895 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:41,760 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:42,834 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:43,737 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:44,630 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:44,781 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:44,841 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:39:45,633 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:39:45,633 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:39:45,782 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:46,815 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:47,850 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:48,830 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:49,645 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:49,798 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:50,861 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:51,865 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:52,835 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:53,821 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:54,695 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:54,903 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:55,927 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:56,890 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:57,931 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:58,904 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:59,730 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:59,884 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:00,640 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:40:00,640 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:40:00,895 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:01,884 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:02,923 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:03,923 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:04,979 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:05,687 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:06,018 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:06,929 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:08,015 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:08,974 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:09,983 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:10,822 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:10,985 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:11,985 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:13,024 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:14,058 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:14,841 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:40:15,069 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:15,661 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:40:15,661 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:40:15,962 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:16,762 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:17,027 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:17,970 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:19,021 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:20,085 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:20,987 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:21,762 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:22,035 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:23,123 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:24,079 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:25,100 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:26,122 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:27,102 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:27,718 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:28,068 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:29,111 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:30,138 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:30,777 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:40:30,777 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:40:31,134 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:32,186 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:32,744 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:33,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:34,259 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:35,217 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:36,216 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:37,198 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:37,790 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:38,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:39,232 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:40,219 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:41,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:42,201 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:42,814 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:43,240 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:44,250 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:44,842 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:40:45,260 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:45,698 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:40:45,698 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:40:46,265 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:47,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:48,274 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:48,798 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:49,282 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:50,282 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:51,315 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:52,329 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:53,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:53,825 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:54,317 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:55,347 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:56,308 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:57,328 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:58,381 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:59,461 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:59,779 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:00,427 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:00,699 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:41:00,700 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:41:01,415 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:02,480 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:03,493 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:04,405 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:04,829 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:05,425 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:06,450 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:07,498 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:08,446 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:09,456 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:10,441 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:10,818 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:11,434 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:12,527 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:13,486 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:14,502 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:14,842 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:41:15,540 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:15,792 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:41:15,792 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:41:15,883 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:16,475 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:17,522 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:18,538 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:19,540 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:20,561 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:20,901 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:21,602 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:22,635 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:23,643 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:24,605 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:25,652 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:26,614 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:26,870 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:27,698 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:28,652 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:29,668 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:30,642 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:30,759 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:41:30,759 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:41:31,712 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:32,726 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:32,862 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:33,654 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:34,682 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:35,695 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:36,769 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:37,802 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:38,735 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:38,837 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:39,754 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:40,697 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:41,752 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:42,741 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:43,767 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:43,850 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:44,778 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:44,842 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:41:45,686 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:41:45,686 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:41:45,785 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:46,845 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:47,779 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:48,870 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:48,890 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:49,850 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:50,780 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:51,817 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:52,840 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:53,888 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:53,911 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:54,827 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:55,845 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:56,881 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:57,830 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:58,839 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:58,921 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:59,837 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:00,750 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:42:00,750 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:42:00,848 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:01,845 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:02,852 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:03,908 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:04,880 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:04,908 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:05,909 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:06,958 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:07,920 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:08,942 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:09,977 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:10,012 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:10,965 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:11,980 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:12,978 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:13,969 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:14,843 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:42:14,978 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:15,760 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:42:15,760 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:42:15,908 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:16,004 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:16,989 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:18,056 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:18,994 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:20,022 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:20,992 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:21,034 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:22,083 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:23,089 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:24,159 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:25,021 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:26,006 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:26,070 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:27,083 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:28,117 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:29,147 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:30,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:30,789 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:42:30,789 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:42:31,164 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:31,948 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:32,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:33,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:34,186 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:35,206 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:36,206 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:36,951 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:37,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:38,231 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:39,194 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:40,272 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:41,306 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:42,016 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:42,303 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:43,368 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:44,302 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:44,843 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:42:45,351 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:45,782 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:42:45,783 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:42:46,351 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:47,023 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:47,334 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:48,397 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:49,395 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:50,377 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:51,363 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:52,024 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:52,422 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:53,399 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:54,387 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:55,379 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:56,369 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:57,030 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:57,395 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:58,446 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:59,414 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:00,461 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:00,816 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:43:00,817 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:43:01,496 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:02,625 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:03,023 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:03,535 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:04,557 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:05,594 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:06,526 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:07,535 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:08,104 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:08,601 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:09,521 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:10,581 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:11,535 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:12,590 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:13,559 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:14,057 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:14,572 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:14,844 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:43:15,603 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:15,778 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:43:15,778 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:43:16,671 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:17,609 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:18,631 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:19,103 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:19,608 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:20,671 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:21,694 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:22,683 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:23,679 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:24,694 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:25,091 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:25,711 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:26,706 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:27,752 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:28,746 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:29,771 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:30,802 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:30,823 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:43:30,824 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:43:30,990 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:31,829 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:32,785 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:33,793 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:34,761 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:35,764 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:36,180 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:36,812 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:37,857 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:38,854 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:39,791 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:40,868 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:41,811 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:42,100 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:42,878 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:43,857 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:44,844 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:43:44,845 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:45,771 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:43:45,771 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:43:45,827 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:46,852 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:47,903 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:48,100 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:48,984 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:49,927 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:50,928 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:51,923 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:52,927 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:53,202 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:53,940 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:54,952 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:55,912 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:56,932 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:57,908 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:58,980 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:59,192 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:00,040 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:00,811 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:44:00,811 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:44:00,977 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:02,021 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:03,042 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:04,056 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:05,010 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:05,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:06,064 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:07,119 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:08,042 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:09,061 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:10,119 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:10,195 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:11,078 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:12,081 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:13,116 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:14,160 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:14,845 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:44:15,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:15,825 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:44:15,825 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:44:15,907 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:16,191 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:17,240 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:18,169 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:19,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:20,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:21,228 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:21,242 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:22,261 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:23,350 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:24,290 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:25,276 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:26,233 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:26,310 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:27,334 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:28,352 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:29,346 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:30,392 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:30,829 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:44:30,829 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:44:31,365 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:31,924 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:32,417 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:33,347 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:34,399 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:35,379 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:36,379 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:37,242 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:37,395 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:38,391 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:39,462 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:40,499 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:41,486 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:42,484 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:43,231 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:43,485 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:44,571 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:44,845 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:44:45,504 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:45,821 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:44:45,821 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:44:46,581 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:47,484 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:48,302 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:48,510 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:49,560 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:50,563 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:51,585 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:52,549 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:53,549 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:54,226 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:54,566 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:55,639 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:56,703 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:57,613 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:58,609 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:59,267 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:59,671 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:00,635 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:00,821 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:45:00,822 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:45:01,613 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:02,612 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:03,702 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:04,303 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:04,659 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:05,644 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:06,726 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:07,754 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:08,708 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:09,663 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:10,254 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:10,717 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:11,725 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:12,691 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:13,761 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:14,732 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:14,846 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:45:15,808 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:15,823 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:45:15,823 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:45:15,895 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:16,721 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:17,696 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:18,729 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:19,737 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:20,731 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:21,396 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:21,816 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:22,841 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:23,806 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:24,767 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:25,774 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:26,837 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:27,341 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:27,823 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:28,803 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:29,869 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:30,834 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:45:30,835 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:45:30,877 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:31,790 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:32,366 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:32,854 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:33,936 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:34,895 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:35,897 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:36,882 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:37,860 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:38,307 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:38,903 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:39,897 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:40,958 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:41,968 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:42,922 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:43,952 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:44,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:44,846 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:45:44,871 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:45,828 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:45:45,828 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:45:45,934 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:46,899 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:47,905 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:48,901 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:49,950 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:50,286 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:50,918 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:51,951 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:52,931 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:53,916 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:54,968 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:55,399 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:55,982 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:56,980 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:57,969 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:58,964 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:00,100 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:00,877 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:46:00,877 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:46:00,959 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:00,995 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:02,031 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:03,019 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:04,049 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:05,043 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:06,071 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:06,295 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:07,147 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:08,165 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:09,076 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:10,076 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:11,074 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:12,131 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:12,278 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:13,135 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:14,147 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:14,846 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:46:15,107 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:15,961 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:46:15,961 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:46:16,154 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:17,168 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:17,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:18,151 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:19,140 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:20,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:21,197 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:22,193 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:23,242 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:23,386 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:24,240 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:25,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:26,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:27,309 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:28,278 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:28,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:29,309 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:30,239 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:30,987 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:46:30,987 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:46:31,282 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:32,285 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:33,431 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:34,320 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:34,365 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:35,396 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:36,397 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:37,366 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:38,388 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:39,385 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:39,462 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:40,445 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:41,510 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:42,447 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:43,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:44,540 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:44,847 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:46:44,848 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:45,516 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:46,053 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:46:46,053 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:46:46,541 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:47,651 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:48,586 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:49,604 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:50,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:50,616 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:51,582 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:52,684 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:53,673 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:54,636 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:55,433 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:55,651 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:56,632 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:57,646 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:58,678 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:59,681 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:00,476 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:00,717 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:01,098 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:47:01,098 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:47:01,825 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:02,767 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:03,721 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:04,736 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:05,478 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:05,738 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:06,796 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:07,744 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:08,749 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:09,855 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:10,493 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:10,778 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:11,833 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:12,802 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:13,847 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:14,847 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:47:14,859 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:15,848 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:15,856 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:16,085 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:47:16,085 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:47:16,823 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:17,790 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:18,822 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:19,856 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:20,847 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:21,452 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:21,873 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:22,840 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:23,816 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:24,872 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:25,858 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:26,572 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:26,863 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:27,846 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:28,829 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:29,838 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:30,938 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:31,065 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:47:31,065 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:47:31,862 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:32,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:32,870 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:33,902 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:34,899 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:35,923 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:36,899 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:37,505 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:37,918 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:38,922 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:39,930 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:40,917 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:41,937 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:42,568 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:42,938 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:44,004 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:44,848 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:47:44,959 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:45,950 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:46,029 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:47:46,030 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:47:47,058 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:48,033 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:48,524 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:49,015 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:49,976 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:51,101 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:52,006 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:53,039 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:53,549 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:54,063 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:55,019 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:56,105 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:57,051 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:58,025 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:58,643 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:59,099 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:00,041 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:01,027 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:48:01,027 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:48:01,114 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:02,170 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:03,143 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:04,121 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:04,586 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:05,129 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:06,166 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:07,144 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:08,273 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:09,162 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:09,612 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:10,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:11,224 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:12,156 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:13,231 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:14,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:14,848 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:48:14,849 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:15,209 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:16,095 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:48:16,095 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:48:16,209 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:17,272 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:18,258 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:19,238 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:20,252 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:20,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:21,205 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:22,296 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:23,228 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:24,238 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:25,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:25,638 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:26,270 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:27,232 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:28,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:29,277 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:30,287 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:30,659 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:31,057 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:48:31,058 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:48:31,387 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:32,313 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:33,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:34,341 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:35,377 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:36,297 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:36,605 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:37,318 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:38,325 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:39,368 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:40,370 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:41,379 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:41,642 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:42,367 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:43,378 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:44,378 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:44,849 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:48:45,457 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:46,086 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:48:46,086 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:48:46,456 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:47,245 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:47,424 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:48,393 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:49,441 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:50,439 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:51,472 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:52,515 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:52,686 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:53,606 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:54,593 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:55,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:56,566 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:57,562 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:58,564 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:58,668 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:59,569 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:00,713 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:01,064 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:49:01,065 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:49:01,602 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:02,596 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:03,616 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:04,641 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:04,680 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:05,625 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:06,674 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:07,622 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:08,650 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:09,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:09,651 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:10,630 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:11,631 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:12,843 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:13,689 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:14,691 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:14,770 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:14,849 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:49:15,738 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:16,057 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:49:16,057 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:49:16,675 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:17,721 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:18,763 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:19,727 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:19,781 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:20,761 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:21,840 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:22,911 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:23,793 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:24,729 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:24,804 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:25,787 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:26,767 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:27,787 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:28,915 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:29,754 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:29,842 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:30,874 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:31,088 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:49:31,088 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:49:31,873 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:32,917 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:33,923 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:34,971 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:35,712 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:35,917 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:36,935 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:37,969 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:38,890 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:39,950 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:40,762 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:40,958 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:41,982 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:43,007 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:43,991 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:44,850 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:49:45,007 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:45,850 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:45,977 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:46,062 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:49:46,062 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:49:46,988 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:48,038 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:49,026 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:50,018 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:51,024 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:51,746 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:52,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:53,094 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:54,157 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:55,113 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:56,126 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:57,144 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:57,724 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:58,218 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:59,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:00,150 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:01,093 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:50:01,093 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:50:01,217 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:02,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:02,871 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:03,301 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:04,269 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:05,241 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:06,247 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:07,343 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:08,376 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:08,850 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:09,277 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:10,242 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:11,245 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:12,247 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:13,281 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:13,868 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:14,342 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:14,850 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:50:15,277 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:16,098 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:50:16,099 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:50:16,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:17,357 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:18,330 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:19,362 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:19,807 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:20,394 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:21,339 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:22,345 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:23,345 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:24,413 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:25,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:25,804 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:26,425 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:27,378 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:28,462 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:29,515 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:30,540 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:30,876 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:31,139 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:50:31,139 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:50:31,458 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:32,469 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:33,475 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:34,480 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:35,560 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:36,122 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:36,583 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:37,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:38,587 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:39,585 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:40,604 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:41,600 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:41,979 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:42,645 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:43,630 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:44,644 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:44,850 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:50:45,673 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:46,127 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:50:46,127 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:50:46,743 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:47,260 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:47,678 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:48,650 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:49,692 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:50,692 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:51,746 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:52,726 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:52,923 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:53,767 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:54,839 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:55,868 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:56,836 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:57,837 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:57,923 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:58,789 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:59,817 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:00,768 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:01,113 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:51:01,113 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:51:01,845 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:02,818 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:02,979 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:03,817 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:04,857 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:05,816 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:06,832 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:07,862 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:08,863 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:08,929 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:09,928 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:10,916 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:12,025 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:13,059 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:13,941 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:13,951 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:14,851 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:51:14,974 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:16,063 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:16,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:51:16,222 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:51:17,023 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:18,013 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:18,954 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:18,990 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:20,015 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:21,046 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:22,078 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:23,125 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:23,969 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:24,074 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:25,129 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:26,134 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:27,105 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:28,147 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:29,167 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:29,966 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:30,208 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:31,153 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:51:31,153 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:51:31,251 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:32,223 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:33,219 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:34,263 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:34,973 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:35,270 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:36,272 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:37,224 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:38,351 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:39,231 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:40,263 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:40,955 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:41,279 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:42,319 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:43,339 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:44,360 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:44,851 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:51:45,340 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:46,165 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:51:46,165 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:51:46,268 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:46,400 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:47,410 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:48,395 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:49,410 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:50,380 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:51,444 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:52,032 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:52,370 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:53,451 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:54,458 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:55,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:56,423 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:57,504 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:57,992 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:58,551 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:59,496 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:00,466 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:01,150 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:52:01,150 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:52:01,474 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:02,495 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:03,029 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:03,524 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:04,583 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:05,534 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:06,533 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:07,626 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:08,030 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:08,516 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:09,482 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:10,541 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:11,544 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:12,503 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:13,048 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:13,576 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:14,521 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:14,852 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:52:15,560 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:16,207 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:52:16,207 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:52:16,540 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:17,574 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:18,558 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:19,027 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:19,609 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:20,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:21,661 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:22,600 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:23,653 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:24,109 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:24,722 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:25,633 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:26,751 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:27,655 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:28,708 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:29,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:29,675 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:30,706 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:31,170 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:52:31,170 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:52:31,671 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:32,719 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:33,758 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:34,735 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:35,124 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:35,754 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:36,720 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:37,714 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:38,723 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:39,723 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:40,133 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:40,747 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:41,796 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:42,764 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:43,771 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:44,799 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:44,852 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:52:45,811 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:45,853 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:46,289 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:52:46,290 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:52:46,745 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:47,767 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:48,811 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:49,784 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:50,805 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:51,146 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:51,827 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:52,809 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:53,876 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:54,868 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:55,887 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:56,290 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:56,873 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:57,885 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:58,913 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:59,928 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:00,973 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:01,333 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:53:01,334 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:53:01,426 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:01,903 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:02,917 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:03,941 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:04,972 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:05,961 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:06,979 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:07,192 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:08,018 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:08,998 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:09,964 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:11,027 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:12,046 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:12,238 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:13,058 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:14,077 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:14,853 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:53:14,997 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:16,045 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:16,220 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:53:16,220 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:53:17,117 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:17,359 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:18,091 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:19,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:20,085 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:21,114 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:22,114 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:23,122 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:23,267 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:24,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:25,125 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:26,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:27,312 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:28,233 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:28,268 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:29,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:30,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:31,217 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:53:31,217 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:53:31,257 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:32,331 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:33,297 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:33,306 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:34,319 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:35,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:36,345 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:37,370 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:38,297 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:38,318 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:39,397 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:40,359 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:41,362 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:42,371 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:43,304 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:43,468 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:44,414 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:44,853 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:53:45,463 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:46,256 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:53:46,256 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:53:46,469 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:47,513 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:48,335 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:48,499 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:49,611 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:50,589 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:51,550 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:52,533 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:53,526 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:54,330 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:54,509 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:55,614 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:56,526 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:57,536 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:58,592 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:59,425 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:59,525 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:00,538 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:01,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:54:01,221 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:54:01,541 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:02,575 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:03,650 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:04,638 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:05,330 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:54:05,586 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:06,585 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:07,568 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:08,644 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:09,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:10,384 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:54:10,648 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:11,718 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:12,694 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:13,693 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:14,699 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:14,853 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:54:15,693 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:15,854 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:54:16,237 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:54:16,237 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:54:16,734 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:17,756 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:18,811 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:19,840 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:20,791 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:21,475 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:54:21,814 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:22,806 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:23,843 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:24,872 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:25,828 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:26,902 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:27,451 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:54:27,889 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:28,889 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:29,890 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:30,837 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:31,180 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:54:31,180 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:54:31,870 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:32,853 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:33,263 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:54:33,848 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:34,965 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:35,946 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:37,035 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:37,945 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:38,175 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:54:38,175 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/video_12_25445a41b5bbda0ed7e2.mp4 with policy now +2025-01-17 15:54:38,217 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:54:38,217 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:54:38,217 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:54:38,218 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:54:38,246 DEBUG SenderThread:2520816 [sender.py:send():379] send: telemetry +2025-01-17 15:54:38,378 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: server_info +2025-01-17 15:54:38,378 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: server_info +2025-01-17 15:54:38,428 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/3i9y75zy-media/videos/eval/video_12_25445a41b5bbda0ed7e2.mp4 +2025-01-17 15:54:38,440 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:54:38,558 DEBUG SenderThread:2520816 [sender.py:send():379] send: exit +2025-01-17 15:54:38,558 INFO SenderThread:2520816 [sender.py:send_exit():586] handling exit code: 0 +2025-01-17 15:54:38,558 INFO SenderThread:2520816 [sender.py:send_exit():588] handling runtime: 2633 +2025-01-17 15:54:38,559 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:54:38,559 INFO SenderThread:2520816 [sender.py:send_exit():594] send defer +2025-01-17 15:54:38,559 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:38,559 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-17 15:54:38,559 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:38,559 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-17 15:54:38,559 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 1 +2025-01-17 15:54:38,559 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:38,559 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-17 15:54:38,560 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:38,560 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-17 15:54:38,560 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 2 +2025-01-17 15:54:38,560 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:38,560 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-17 15:54:38,560 INFO HandlerThread:2520816 [system_monitor.py:finish():203] Stopping system monitor +2025-01-17 15:54:38,560 DEBUG SystemMonitor:2520816 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-17 15:54:38,560 INFO HandlerThread:2520816 [interfaces.py:finish():200] Joined cpu monitor +2025-01-17 15:54:38,560 DEBUG SystemMonitor:2520816 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-17 15:54:38,560 INFO HandlerThread:2520816 [interfaces.py:finish():200] Joined disk monitor +2025-01-17 15:54:38,614 INFO HandlerThread:2520816 [interfaces.py:finish():200] Joined gpu monitor +2025-01-17 15:54:38,614 INFO HandlerThread:2520816 [interfaces.py:finish():200] Joined memory monitor +2025-01-17 15:54:38,614 INFO HandlerThread:2520816 [interfaces.py:finish():200] Joined network monitor +2025-01-17 15:54:38,615 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:38,615 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-17 15:54:38,615 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 3 +2025-01-17 15:54:38,615 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:54:38,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:38,615 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-17 15:54:38,615 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:38,615 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-17 15:54:38,615 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 4 +2025-01-17 15:54:38,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:38,615 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-17 15:54:38,777 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:54:38,777 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/video_12_25445a41b5bbda0ed7e2.mp4 +2025-01-17 15:54:38,778 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:54:39,036 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/aciuk6x1-media/videos/eval/video_12_25445a41b5bbda0ed7e2.mp4 +2025-01-17 15:54:40,778 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:54:45,075 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:54:45,075 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:54:45,075 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: keepalive +2025-01-17 15:54:45,075 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:45,076 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-17 15:54:45,076 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 5 +2025-01-17 15:54:45,076 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:54:45,076 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:54:45,076 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:54:45,078 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:54:45,078 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:45,078 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-17 15:54:45,083 DEBUG SenderThread:2520816 [sender.py:send():379] send: summary +2025-01-17 15:54:45,083 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:54:45,084 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:45,084 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-17 15:54:45,084 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 6 +2025-01-17 15:54:45,084 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:45,084 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-17 15:54:45,084 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:45,084 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-17 15:54:45,088 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:54:45,213 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 7 +2025-01-17 15:54:45,214 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:45,214 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-17 15:54:45,214 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:45,214 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-17 15:54:45,740 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:54:45,780 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/config.yaml +2025-01-17 15:54:45,780 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:54:47,267 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 8 +2025-01-17 15:54:47,267 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:54:47,268 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:47,268 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-17 15:54:47,268 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:47,268 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-17 15:54:47,268 INFO SenderThread:2520816 [job_builder.py:build():440] Attempting to build job artifact +2025-01-17 15:54:47,270 INFO SenderThread:2520816 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-17 15:54:47,289 INFO SenderThread:2520816 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-17 15:54:47,302 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 9 +2025-01-17 15:54:47,302 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:47,302 DEBUG SenderThread:2520816 [sender.py:send():379] send: artifact +2025-01-17 15:54:47,302 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-17 15:54:47,781 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:54:47,809 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:54:47,906 INFO wandb-upload_3:2520816 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzmv2qlke +2025-01-17 15:54:47,906 INFO wandb-upload_1:2520816 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpxqk64tfw +2025-01-17 15:54:48,115 INFO wandb-upload_0:2520816 [upload_job.py:push():88] Uploaded file /tmp/tmp7cvwy9fu/wandb-job.json +2025-01-17 15:54:48,756 INFO SenderThread:2520816 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ0OTIyOTkxNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ0OTEwNDY1OA==', 'versionIndex': 442}}} +2025-01-17 15:54:48,756 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:48,756 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-17 15:54:48,756 INFO SenderThread:2520816 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-17 15:54:48,782 INFO SenderThread:2520816 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files +2025-01-17 15:54:48,782 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/config.yaml config.yaml +2025-01-17 15:54:48,782 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json wandb-summary.json +2025-01-17 15:54:48,782 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:54:48,785 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/requirements.txt requirements.txt +2025-01-17 15:54:48,785 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-metadata.json wandb-metadata.json +2025-01-17 15:54:48,785 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 15:54:48,785 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log output.log +2025-01-17 15:54:48,786 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/diff.patch diff.patch +2025-01-17 15:54:48,788 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/video_12_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_12_25445a41b5bbda0ed7e2.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/2_eval_2_a9b669c92c8beee6656c.mp4 media/videos/eval/2_eval_2_a9b669c92c8beee6656c.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/5_eval_5_5eccb1be08111ca14e10.mp4 media/videos/eval/5_eval_5_5eccb1be08111ca14e10.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/1_eval_1_fe99acb9f5fbe6b6c23e.mp4 media/videos/eval/1_eval_1_fe99acb9f5fbe6b6c23e.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/7_eval_7_eab2215188c302c460ba.mp4 media/videos/eval/7_eval_7_eab2215188c302c460ba.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/9_eval_9_9a2233becb7986a47e63.mp4 media/videos/eval/9_eval_9_9a2233becb7986a47e63.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/6_eval_6_5f3fdb589a9c504e4946.mp4 media/videos/eval/6_eval_6_5f3fdb589a9c504e4946.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/11_eval_11_f83167404e8bb0859778.mp4 media/videos/eval/11_eval_11_f83167404e8bb0859778.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/3_eval_3_2988eff819b475f13199.mp4 media/videos/eval/3_eval_3_2988eff819b475f13199.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/10_eval_10_dc26db5ffbb288ebf69b.mp4 media/videos/eval/10_eval_10_dc26db5ffbb288ebf69b.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/8_eval_8_50d0e7e81a609c529244.mp4 media/videos/eval/8_eval_8_50d0e7e81a609c529244.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/4_eval_4_8487a970815421048be9.mp4 media/videos/eval/4_eval_4_8487a970815421048be9.mp4 +2025-01-17 15:54:48,790 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/0_eval_0_98c68d418db5b73df1b0.mp4 media/videos/eval/0_eval_0_98c68d418db5b73df1b0.mp4 +2025-01-17 15:54:48,790 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 10 +2025-01-17 15:54:48,790 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:54:48,790 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:48,790 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-17 15:54:48,790 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:48,790 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-17 15:54:48,790 INFO SenderThread:2520816 [file_pusher.py:finish():169] shutting down file pusher +2025-01-17 15:54:48,862 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:54:48,862 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:54:48,976 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/config.yaml +2025-01-17 15:54:49,024 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:54:49,083 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:54:49,086 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/requirements.txt +2025-01-17 15:54:49,286 INFO Thread-11 (_thread_body):2520816 [sender.py:transition_state():614] send defer: 11 +2025-01-17 15:54:49,286 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:49,286 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-17 15:54:49,287 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:49,287 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-17 15:54:49,287 INFO SenderThread:2520816 [file_pusher.py:join():175] waiting for file pusher +2025-01-17 15:54:49,289 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 12 +2025-01-17 15:54:49,289 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:49,289 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-17 15:54:49,289 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:49,289 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-17 15:54:49,289 INFO SenderThread:2520816 [file_stream.py:finish():601] file stream finish called +2025-01-17 15:54:49,488 INFO SenderThread:2520816 [file_stream.py:finish():605] file stream finish is done +2025-01-17 15:54:49,488 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 13 +2025-01-17 15:54:49,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:49,488 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-17 15:54:49,488 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:49,488 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-17 15:54:49,488 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 14 +2025-01-17 15:54:49,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:49,488 DEBUG SenderThread:2520816 [sender.py:send():379] send: final +2025-01-17 15:54:49,488 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-17 15:54:49,488 DEBUG SenderThread:2520816 [sender.py:send():379] send: footer +2025-01-17 15:54:49,489 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:49,489 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-17 15:54:49,602 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:54:49,602 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:54:49,802 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:49,883 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: get_summary +2025-01-17 15:54:49,973 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-17 15:54:50,009 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: shutdown +2025-01-17 15:54:50,010 INFO HandlerThread:2520816 [handler.py:finish():882] shutting down handler +2025-01-17 15:54:50,602 INFO WriterThread:2520816 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/run-h18xe90d.wandb +2025-01-17 15:54:50,602 INFO SenderThread:2520816 [sender.py:finish():1615] shutting down sender +2025-01-17 15:54:50,602 INFO SenderThread:2520816 [file_pusher.py:finish():169] shutting down file pusher +2025-01-17 15:54:50,603 INFO SenderThread:2520816 [file_pusher.py:join():175] waiting for file pusher diff --git a/h18xe90d/wandb/debug.log b/h18xe90d/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f16ebfd10e704f198b497772cb10ee7531c3c918 --- /dev/null +++ b/h18xe90d/wandb/debug.log @@ -0,0 +1,37 @@ +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_setup.py:_flush():76] Configure stats pid to 2520639 +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/settings +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/logs/debug.log +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/logs/debug-internal.log +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_init.py:init():569] calling init triggers +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3456_0', 'wandb_notes': '3456_0_req_3921_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d'} +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_init.py:init():619] starting backend +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_init.py:init():623] setting up manager +2025-01-17 15:10:44,430 INFO MainThread:2520639 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-17 15:10:44,432 INFO MainThread:2520639 [wandb_init.py:init():631] backend started and connected +2025-01-17 15:10:44,447 INFO MainThread:2520639 [wandb_init.py:init():720] updated telemetry +2025-01-17 15:10:44,454 INFO MainThread:2520639 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-17 15:10:44,753 INFO MainThread:2520639 [wandb_run.py:_on_init():2435] communicating current version +2025-01-17 15:10:44,796 INFO MainThread:2520639 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-17 15:10:44,797 INFO MainThread:2520639 [wandb_init.py:init():804] starting run threads in backend +2025-01-17 15:10:45,171 INFO MainThread:2520639 [wandb_run.py:_console_start():2413] atexit reg +2025-01-17 15:10:45,171 INFO MainThread:2520639 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-17 15:10:45,171 INFO MainThread:2520639 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-17 15:10:45,172 INFO MainThread:2520639 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-17 15:10:45,173 INFO MainThread:2520639 [wandb_init.py:init():847] run started, returning control to user process +2025-01-17 15:10:45,173 INFO MainThread:2520639 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/tb, True +2025-01-17 15:10:51,873 INFO MainThread:2520639 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-17 15:54:38,211 INFO MainThread:2520639 [wandb_run.py:_finish():2107] finishing run serialexperimentsleon/extrinsic_contact_downstream/h18xe90d +2025-01-17 15:54:38,263 INFO MainThread:2520639 [wandb_run.py:_atexit_cleanup():2374] got exitcode: 0 +2025-01-17 15:54:38,310 INFO MainThread:2520639 [wandb_run.py:_restore():2352] restore +2025-01-17 15:54:38,310 INFO MainThread:2520639 [wandb_run.py:_restore():2358] restore done +2025-01-17 15:54:51,019 INFO MainThread:2520639 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-17 15:54:51,043 INFO MainThread:2520639 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-17 15:54:52,124 INFO MainThread:2520639 [wandb_run.py:_footer_sync_info():3975] logging synced files diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/code/FISH/eval_robot.py b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/config.yaml b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3764629c7049000ab9295141be6d9a9ab904d189 --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/config.yaml @@ -0,0 +1,927 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '3456_0' +wandb_notes: + desc: null + value: 3456_0_req_3921_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: null +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 20 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737144644 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 2 + - 14 + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/diff.patch b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..9505998782078316d897b259427f75a7fd990633 --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/diff.patch @@ -0,0 +1,78 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0b377ae..a6b1973 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -132,8 +132,11 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1000_0' # all + + # all books, 10/20 demos per book ++# crp_D+act history ++wandb_run_id: '3456_0' ++# wandb_run_id: '3456_1' # seed 1 ++ + # RGBD+mask+act history +-wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..d9b15a5 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,12 +167,14 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ # '3456_0', ++ '3456_1', + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/0_eval_0_98c68d418db5b73df1b0.mp4 b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/0_eval_0_98c68d418db5b73df1b0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..64c693609e2342d36a0cd2a7c8616844e0f1fffb --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/0_eval_0_98c68d418db5b73df1b0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98c68d418db5b73df1b05761452a845f5a35493133d7805bbed1dc89315be5c9 +size 1167172 diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/10_eval_10_dc26db5ffbb288ebf69b.mp4 b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/10_eval_10_dc26db5ffbb288ebf69b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0a260c3acaed818b2d422f24013ddd358abb40a4 --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/10_eval_10_dc26db5ffbb288ebf69b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc26db5ffbb288ebf69be3b37f1a149b05a67b55add127df99546db602a0c6ce +size 1385355 diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/11_eval_11_f83167404e8bb0859778.mp4 b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/11_eval_11_f83167404e8bb0859778.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3fac08fcc76564381b2f34b9547dd443767d1db8 --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/11_eval_11_f83167404e8bb0859778.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f83167404e8bb0859778ab785b3ec72f319922e32ba8be47628685d52a3fb45e +size 1451754 diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/1_eval_1_fe99acb9f5fbe6b6c23e.mp4 b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/1_eval_1_fe99acb9f5fbe6b6c23e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..34ad75c54a1653fae4b114452d1c514f6ed4ba4e --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/1_eval_1_fe99acb9f5fbe6b6c23e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe99acb9f5fbe6b6c23e6a1f518986a970f45769de1d4acd955b9b5f9bb5e789 +size 1154808 diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/2_eval_2_a9b669c92c8beee6656c.mp4 b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/2_eval_2_a9b669c92c8beee6656c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..520b3d7d70b24559080ec2d1f13db217919ded5e --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/2_eval_2_a9b669c92c8beee6656c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9b669c92c8beee6656cacd11f615b7c5aee743d336e4b927a08eddfab14d19b +size 1163094 diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/3_eval_3_2988eff819b475f13199.mp4 b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/3_eval_3_2988eff819b475f13199.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9f53218ead20944215e79392c1f28fb1a356bfa0 --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/3_eval_3_2988eff819b475f13199.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2988eff819b475f131997a6481c4c4ca6d5d6a9124cea83e7998596e71612259 +size 1160636 diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/4_eval_4_8487a970815421048be9.mp4 b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/4_eval_4_8487a970815421048be9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cd10d13f869460cd7ad5876af3d21b2804f258b7 --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/4_eval_4_8487a970815421048be9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8487a970815421048be960754b4caeed67e948d55be1d9121a1fb0628f53fb10 +size 1171773 diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/5_eval_5_5eccb1be08111ca14e10.mp4 b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/5_eval_5_5eccb1be08111ca14e10.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..319a5d82563cfbfa0ac652124a2a0660e640e2d5 --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/5_eval_5_5eccb1be08111ca14e10.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5eccb1be08111ca14e10f50c8b05a5c13c37a164fba6aaa66d39310d8cfd12f3 +size 1269853 diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/6_eval_6_5f3fdb589a9c504e4946.mp4 b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/6_eval_6_5f3fdb589a9c504e4946.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2226ee9574a1583ee0291f3b484e7bdcc7facfad --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/6_eval_6_5f3fdb589a9c504e4946.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f3fdb589a9c504e49465abf5578b1e48353445b5fb7da025d1597ab32b5a0b7 +size 1231149 diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/7_eval_7_eab2215188c302c460ba.mp4 b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/7_eval_7_eab2215188c302c460ba.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df5ecd54236202a99e4f21523217aca7b73a21e7 --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/7_eval_7_eab2215188c302c460ba.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eab2215188c302c460ba00e7f817b9602d979b290be42c792b2cff46a93ae595 +size 1153200 diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/8_eval_8_50d0e7e81a609c529244.mp4 b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/8_eval_8_50d0e7e81a609c529244.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..015f40bfeb7f352813b6a1f3479ea850141af1b1 --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/8_eval_8_50d0e7e81a609c529244.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50d0e7e81a609c529244b4bf0955c640353908a3899746c56f1df81d8ea406b8 +size 1174619 diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/9_eval_9_9a2233becb7986a47e63.mp4 b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/9_eval_9_9a2233becb7986a47e63.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6f859937d7908d50dc10a366080f5f19e086083c --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/9_eval_9_9a2233becb7986a47e63.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a2233becb7986a47e63b28b21334780efa49c26963ca4e624fb1db5d670d747 +size 1362409 diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/video_12_25445a41b5bbda0ed7e2.mp4 b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/video_12_25445a41b5bbda0ed7e2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa868fbb75c51fb59ae42ff30716054804074343 Binary files /dev/null and b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/video_12_25445a41b5bbda0ed7e2.mp4 differ diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..3ec0a7938b7e360f30a0e8fe96fe905766ed9a04 --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log @@ -0,0 +1,624 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737144651.874398]: resetting environment +[INFO] [1737144651.878342]: cleared current plan +[INFO] [1737144651.879937]: moving to home +[INFO] [1737144652.885069]: reached home +[INFO] [1737144652.886333]: reset action history +[INFO] [1737144655.595084]: environment reset +Starting episode 0 +[INFO] [1737144655.598425]: resetting environment +[INFO] [1737144655.601082]: cleared current plan +[INFO] [1737144655.601864]: moving to home +[INFO] [1737144656.604273]: reached home +[INFO] [1737144656.606447]: reset action history +[INFO] [1737144659.315314]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1737144693.361197]: resetting environment +[INFO] [1737144693.365009]: cleared current plan +[INFO] [1737144693.366265]: moving to home +[INFO] [1737144694.368660]: reached home +[INFO] [1737144694.370304]: reset action history +[INFO] [1737144697.078603]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/episode_rosbags/episode_0_2025-01-17-15-11-37.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2520639.log +started roslaunch server http://158.130.50.37:40717/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2520945] +process[print_text-2]: started with pid [2520969] +process[rosbag_record-3]: started with pid [2520970] +started bagging! +[INFO] [1737144724.066419]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1737144724.805359]: Stored episode 1. +[INFO] [1737144724.806252]: Saving video... +[INFO] [1737144724.992316]: Video saved! +Starting episode 1 +[INFO] [1737144725.018863]: resetting environment +[INFO] [1737144725.030731]: cleared current plan +[INFO] [1737144725.031056]: moving to home +[INFO] [1737144728.633262]: reached home +[INFO] [1737144728.633700]: reset action history +[INFO] [1737144731.341347]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737144744.681671]: resetting environment +[INFO] [1737144744.685477]: cleared current plan +[INFO] [1737144744.685823]: moving to home +[INFO] [1737144745.688077]: reached home +[INFO] [1737144745.688575]: reset action history +[INFO] [1737144748.394022]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/episode_rosbags/episode_1_2025-01-17-15-12-28.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2520639.log +started roslaunch server http://158.130.50.37:46563/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2521090] +process[print_text-5]: started with pid [2521091] +process[rosbag_record-6]: started with pid [2521115] +started bagging! +For topic gripper_width: timestamp difference is 122833191 for nearest: 1737144748977284371 - target: 1737144748854451180 at idx 10 +For topic gripper_width: timestamp difference is -108415682 for nearest: 1737144751377231281 - target: 1737144751485646963 at idx 38 +For topic gripper_width: timestamp difference is -116690361 for nearest: 1737144754377282656 - target: 1737144754493973017 at idx 80 +For topic gripper_width: timestamp difference is 104710130 for nearest: 1737144764977276115 - target: 1737144764872565985 at idx 222 +For topic gripper_width: timestamp difference is 109381331 for nearest: 1737144773377322806 - target: 1737144773267941475 at idx 249 +[INFO] [1737144775.351117]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1737144776.129943]: Stored episode 2. +[INFO] [1737144776.130370]: Saving video... +[INFO] [1737144776.591710]: Video saved! +Starting episode 2 +[INFO] [1737144776.635984]: resetting environment +[INFO] [1737144776.658639]: cleared current plan +[INFO] [1737144776.659009]: moving to home +[INFO] [1737144780.265245]: reached home +[INFO] [1737144780.266698]: reset action history +[INFO] [1737144783.057635]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737144787.347223]: resetting environment +[INFO] [1737144787.365821]: cleared current plan +[INFO] [1737144787.367276]: moving to home +[INFO] [1737144788.376415]: reached home +[INFO] [1737144788.379158]: reset action history +[INFO] [1737144791.137455]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/episode_rosbags/episode_2_2025-01-17-15-13-11.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2520639.log +started roslaunch server http://158.130.50.37:42745/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2521247] +process[print_text-8]: started with pid [2521248] +process[rosbag_record-9]: started with pid [2521249] +started bagging! +For topic gripper_width: timestamp difference is 102115072 for nearest: 1737144805644025959 - target: 1737144805541910887 at idx 203 +[INFO] [1737144818.210732]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1737144819.014337]: Stored episode 3. +[INFO] [1737144819.015495]: Saving video... +[INFO] [1737144819.303401]: Video saved! +Starting episode 3 +[INFO] [1737144819.361428]: resetting environment +[INFO] [1737144819.395255]: cleared current plan +[INFO] [1737144819.409643]: moving to home +[INFO] [1737144823.019958]: reached home +[INFO] [1737144823.025064]: reset action history +[INFO] [1737144825.814686]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737144828.693485]: resetting environment +[INFO] [1737144828.710365]: cleared current plan +[INFO] [1737144828.712851]: moving to home +[INFO] [1737144829.720586]: reached home +[INFO] [1737144829.726746]: reset action history +[INFO] [1737144832.502596]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/episode_rosbags/episode_3_2025-01-17-15-13-52.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2520639.log +started roslaunch server http://158.130.50.37:42169/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2521399] +process[print_text-11]: started with pid [2521400] +process[rosbag_record-12]: started with pid [2521401] +started bagging! +For topic gripper_width: timestamp difference is 127025438 for nearest: 1737144836643933607 - target: 1737144836516908169 at idx 63 +For topic gripper_width: timestamp difference is 100162523 for nearest: 1737144844843937414 - target: 1737144844743774891 at idx 187 +For topic gripper_width: timestamp difference is 100863805 for nearest: 1737144845643983951 - target: 1737144845543120146 at idx 197 +For topic gripper_width: timestamp difference is 102171588 for nearest: 1737144847243968654 - target: 1737144847141797066 at idx 215 +For topic gripper_width: timestamp difference is 103637261 for nearest: 1737144849043936534 - target: 1737144848940299273 at idx 241 +For topic gripper_width: timestamp difference is 103803306 for nearest: 1737144849243934780 - target: 1737144849140131474 at idx 243 +[INFO] [1737144859.623065]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit[pub_text-10] killing on exit +[INFO] [1737144860.440137]: Stored episode 4. +[INFO] [1737144860.446972]: Saving video... +[INFO] [1737144861.068748]: Video saved! +[WARN] [1737144860.981685]: Plan exhausted +[WARN] [1737144861.017180]: Plan exhausted +[WARN] [1737144861.057405]: Plan exhausted +[WARN] [1737144861.097440]: Plan exhausted +[WARN] [1737144861.137571]: Plan exhausted +Starting episode 4 +[INFO] [1737144861.128443]: resetting environment +[INFO] [1737144861.154957]: cleared current plan +[INFO] [1737144861.155254]: moving to home +[INFO] [1737144864.758511]: reached home +[INFO] [1737144864.775942]: reset action history +[INFO] [1737144867.651920]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737144870.177750]: resetting environment +[INFO] [1737144870.206373]: cleared current plan +[INFO] [1737144870.208797]: moving to home +[INFO] [1737144871.218974]: reached home +[INFO] [1737144871.232475]: reset action history +[INFO] [1737144874.081341]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/episode_rosbags/episode_4_2025-01-17-15-14-34.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2520639.log +started roslaunch server http://158.130.50.37:42111/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2521534] +process[print_text-14]: started with pid [2521535] +process[rosbag_record-15]: started with pid [2521536] +started bagging! +For topic gripper_width: timestamp difference is 100150195 for nearest: 1737144885943954554 - target: 1737144885843804359 at idx 159 +For topic gripper_width: timestamp difference is 100950800 for nearest: 1737144886943955027 - target: 1737144886843004227 at idx 170 +For topic gripper_width: timestamp difference is 103421547 for nearest: 1737144889943934061 - target: 1737144889840512514 at idx 207 +For topic gripper_width: timestamp difference is 103539841 for nearest: 1737144890143888371 - target: 1737144890040348530 at idx 209 +For topic gripper_width: timestamp difference is 104235734 for nearest: 1737144890943932141 - target: 1737144890839696407 at idx 220 +For topic gripper_width: timestamp difference is 105037751 for nearest: 1737144891943931403 - target: 1737144891838893652 at idx 235 +For topic gripper_width: timestamp difference is 107436470 for nearest: 1737144894943933300 - target: 1737144894836496830 at idx 249 +For topic gripper_width: timestamp difference is 108077478 for nearest: 1737144895743936491 - target: 1737144895635859013 at idx 249 +For topic gripper_width: timestamp difference is 108564394 for nearest: 1737144896343935268 - target: 1737144896235370874 at idx 249 +For topic gripper_width: timestamp difference is 108768472 for nearest: 1737144896543974885 - target: 1737144896435206413 at idx 249 +For topic gripper_width: timestamp difference is 110100387 for nearest: 1737144898143964170 - target: 1737144898033863783 at idx 249 +For topic gripper_width: timestamp difference is 110900686 for nearest: 1737144899143932388 - target: 1737144899033031702 at idx 249 +For topic gripper_width: timestamp difference is 111068984 for nearest: 1737144899343934794 - target: 1737144899232865810 at idx 249 +[INFO] [1737144901.327023]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1737144902.146973]: Stored episode 5. +[INFO] [1737144902.148820]: Saving video... +[INFO] [1737144902.427538]: Video saved! +Starting episode 5 +[INFO] [1737144902.514823]: resetting environment +[INFO] [1737144902.555320]: cleared current plan +[INFO] [1737144902.560683]: moving to home +[INFO] [1737144906.188447]: reached home +[INFO] [1737144906.198904]: reset action history +[INFO] [1737144909.099916]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737144914.671863]: resetting environment +[INFO] [1737144914.695052]: cleared current plan +[INFO] [1737144914.709739]: moving to home +[INFO] [1737144915.736725]: reached home +[INFO] [1737144915.743742]: reset action history +[INFO] [1737144918.610169]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/episode_rosbags/episode_5_2025-01-17-15-15-18.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2520639.log +started roslaunch server http://158.130.50.37:37689/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-16]: started with pid [2521670] +process[print_text-17]: started with pid [2521671] +process[rosbag_record-18]: started with pid [2521672] +started bagging! +[INFO] [1737144948.670593]: Storing episode... +[pub_text-16] killing on exit +[rosbag_record-18] killing on exit[print_text-17] killing on exit +[WARN] [1737144950.018654]: Plan exhausted +[WARN] [1737144950.058159]: Plan exhausted +[INFO] [1737144949.470788]: Stored episode 6. +[INFO] [1737144949.479733]: Saving video... +[INFO] [1737144949.917473]: Video saved! +Starting episode 6 +[INFO] [1737144950.024127]: resetting environment +[INFO] [1737144950.062745]: cleared current plan +[INFO] [1737144950.072247]: moving to home +[INFO] [1737144953.696579]: reached home +[INFO] [1737144953.708251]: reset action history +[INFO] [1737144956.637765]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737144958.858764]: resetting environment +[INFO] [1737144958.881171]: cleared current plan +[INFO] [1737144958.884510]: moving to home +[INFO] [1737144959.899440]: reached home +[INFO] [1737144959.908894]: reset action history +[INFO] [1737144962.842039]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/episode_rosbags/episode_6_2025-01-17-15-16-02.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2520639.log +started roslaunch server http://158.130.50.37:35197/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-19]: started with pid [2521812] +process[print_text-20]: started with pid [2521813] +process[rosbag_record-21]: started with pid [2521814] +started bagging! +[INFO] [1737144991.459963]: Storing episode... +[pub_text-19] killing on exit +[print_text-20] killing on exit +[rosbag_record-21] killing on exit +[INFO] [1737144992.269975]: Stored episode 7. +[INFO] [1737144992.278912]: Saving video... +[INFO] [1737144992.956867]: Video saved! +Starting episode 7 +[INFO] [1737144993.054455]: resetting environment +[INFO] [1737144993.104906]: cleared current plan +[INFO] [1737144993.106308]: moving to home +[WARN] [1737144992.816618]: Plan exhausted +[WARN] [1737144992.865908]: Plan exhausted +[WARN] [1737144992.898058]: Plan exhausted +[WARN] [1737144992.939139]: Plan exhausted +[WARN] [1737144992.977376]: Plan exhausted +[WARN] [1737144993.017725]: Plan exhausted +[WARN] [1737144993.056735]: Plan exhausted +[WARN] [1737144993.098705]: Plan exhausted +[INFO] [1737144996.719995]: reached home +[INFO] [1737144996.734575]: reset action history +[INFO] [1737144999.633996]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737145002.125946]: resetting environment +[INFO] [1737145002.172077]: cleared current plan +[INFO] [1737145002.188179]: moving to home +[INFO] [1737145003.201502]: reached home +[INFO] [1737145003.205266]: reset action history +[INFO] [1737145006.073722]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/episode_rosbags/episode_7_2025-01-17-15-16-46.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2520639.log +started roslaunch server http://158.130.50.37:37481/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-22]: started with pid [2521955] +process[print_text-23]: started with pid [2521956] +process[rosbag_record-24]: started with pid [2521957] +started bagging! +For topic gripper_width: timestamp difference is -107367558 for nearest: 1737145011877275901 - target: 1737145011984643459 at idx 86 +[INFO] [1737145033.231937]: Storing episode... +[WARN] [1737145034.577706]: Plan exhausted +[rosbag_record-24] killing on exit +[print_text-23] killing on exit +[pub_text-22] killing on exit +[INFO] [1737145034.144988]: Stored episode 8. +[INFO] [1737145034.168500]: Saving video... +[INFO] [1737145034.391199]: Video saved! +Starting episode 8 +[INFO] [1737145034.540071]: resetting environment +[INFO] [1737145034.595079]: cleared current plan +[INFO] [1737145034.595537]: moving to home +[INFO] [1737145038.211694]: reached home +[INFO] [1737145038.247201]: reset action history +[INFO] [1737145041.308724]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737145044.275025]: resetting environment +[INFO] [1737145044.341135]: cleared current plan +[INFO] [1737145044.367557]: moving to home +[INFO] [1737145045.373464]: reached home +[INFO] [1737145045.375096]: reset action history +[INFO] [1737145048.382568]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/episode_rosbags/episode_8_2025-01-17-15-17-28.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2520639.log +started roslaunch server http://158.130.50.37:33607/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-25]: started with pid [2522070] +process[print_text-26]: started with pid [2522071] +process[rosbag_record-27]: started with pid [2522095] +started bagging! +[INFO] [1737145075.392711]: Storing episode... +[rosbag_record-27] killing on exit +[pub_text-25] killing on exit +[print_text-26] killing on exit +[INFO] [1737145076.232386]: Stored episode 9. +[INFO] [1737145076.247170]: Saving video... +[INFO] [1737145076.588588]: Video saved! +Starting episode 9 +[INFO] [1737145076.718770]: resetting environment +[INFO] [1737145076.743717]: cleared current plan +[INFO] [1737145076.753381]: moving to home +[WARN] [1737145076.736612]: Plan exhausted +[INFO] [1737145080.378605]: reached home +[INFO] [1737145080.421814]: reset action history +[INFO] [1737145083.473094]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737145088.131793]: resetting environment +[INFO] [1737145088.171527]: cleared current plan +[INFO] [1737145088.178016]: moving to home +[INFO] [1737145089.202218]: reached home +[INFO] [1737145089.205651]: reset action history +[INFO] [1737145092.263952]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/episode_rosbags/episode_9_2025-01-17-15-18-12.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2520639.log +started roslaunch server http://158.130.50.37:34915/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-28]: started with pid [2522237] +process[print_text-29]: started with pid [2522238] +process[rosbag_record-30]: started with pid [2522239] +started bagging! +For topic gripper_width: timestamp difference is 107068812 for nearest: 1737145105743950402 - target: 1737145105636881590 at idx 192 +For topic gripper_width: timestamp difference is -178214787 for nearest: 1737145112677224161 - target: 1737145112855438948 at idx 249 +[INFO] [1737145124.718442]: Storing episode... +[rosbag_record-30] killing on exit +[pub_text-28] killing on exit +[print_text-29] killing on exit +[INFO] [1737145125.519492]: Stored episode 10. +[INFO] [1737145125.535129]: Saving video... +[INFO] [1737145125.823667]: Video saved! +Starting episode 10 +[INFO] [1737145125.939959]: resetting environment +[INFO] [1737145125.978858]: cleared current plan +[INFO] [1737145125.986184]: moving to home +[INFO] [1737145129.605814]: reached home +[INFO] [1737145129.624259]: reset action history +[INFO] [1737145132.604269]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737145136.377097]: resetting environment +[INFO] [1737145136.435439]: cleared current plan +[INFO] [1737145136.471512]: moving to home +[INFO] [1737145137.494685]: reached home +[INFO] [1737145137.500746]: reset action history +[INFO] [1737145140.463413]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/episode_rosbags/episode_10_2025-01-17-15-19-00.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2520639.log +started roslaunch server http://158.130.50.37:42585/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-31]: started with pid [2522372] +process[print_text-32]: started with pid [2522373] +process[rosbag_record-33]: started with pid [2522374] +started bagging! +[INFO] [1737145173.301405]: Storing episode... +[rosbag_record-33] killing on exit +[print_text-32] killing on exit +[pub_text-31] killing on exit +[WARN] [1737145174.665358]: Plan exhausted +[WARN] [1737145174.696849]: Plan exhausted +[INFO] [1737145174.137834]: Stored episode 11. +[INFO] [1737145174.163425]: Saving video... +[INFO] [1737145174.501473]: Video saved! +Starting episode 11 +[INFO] [1737145174.671042]: resetting environment +[INFO] [1737145174.725233]: cleared current plan +[INFO] [1737145174.737234]: moving to home +[INFO] [1737145178.345127]: reached home +[INFO] [1737145178.372178]: reset action history +[INFO] [1737145181.484578]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737145184.819521]: resetting environment +[INFO] [1737145184.857528]: cleared current plan +[INFO] [1737145184.859815]: moving to home +[INFO] [1737145185.871459]: reached home +[INFO] [1737145185.875200]: reset action history +[INFO] [1737145188.981676]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/episode_rosbags/episode_11_2025-01-17-15-19-48.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2520639.log +started roslaunch server http://158.130.50.37:40201/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-34]: started with pid [2522506] +process[print_text-35]: started with pid [2522507] +process[rosbag_record-36]: started with pid [2522508] +started bagging! +For topic gripper_width: timestamp difference is 100755753 for nearest: 1737145216743958296 - target: 1737145216643202543 at idx 249 +[INFO] [1737145223.547758]: Storing episode... +[rosbag_record-36] killing on exit +[pub_text-34] killing on exit +[print_text-35] killing on exit +[INFO] [1737145224.317056]: Stored episode 12. +[INFO] [1737145224.340974]: Saving video... +[INFO] [1737145224.863483]: Video saved! +Starting episode 12 +[INFO] [1737145224.989911]: resetting environment +[INFO] [1737145225.001977]: cleared current plan +[INFO] [1737145225.002868]: moving to home +[WARN] [1737145224.896756]: Plan exhausted +[WARN] [1737145224.940833]: Plan exhausted +[WARN] [1737145224.976584]: Plan exhausted +[INFO] [1737145228.613642]: reached home +[INFO] [1737145228.621561]: reset action history +[INFO] [1737145231.765373]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +program exiting gracefully +proceeding to start episode! +[INFO] [1737147277.696025]: Saving video... +[INFO] [1737147277.783431]: Video saved! \ No newline at end of file diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/requirements.txt b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch new file mode 100644 index 0000000000000000000000000000000000000000..ed960f9ae1689f11a78f267e474d398511995733 --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch @@ -0,0 +1,95 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..a6b1973 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 5 ++num_eval: 20 + + debug_timestamps: False + open_loop: False +@@ -132,15 +132,18 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1000_0' # all + + # all books, 10/20 demos per book ++# crp_D+act history ++wandb_run_id: '3456_0' ++# wandb_run_id: '3456_1' # seed 1 ++ + # RGBD+mask+act history +-# wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..d9b15a5 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,12 +167,14 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ # '3456_0', ++ '3456_1', + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-metadata.json b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..07683700f6a55de5f176b98dc3078877655f66ed --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-17T20:10:44.824913", + "startedAt": "2025-01-17T20:10:44.420748", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "ec42957dc10d08dbd8b134558fcd8f9e470ccc31" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3952.85125, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4500.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4530.086, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 426.36322021484375 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4ff4ffee774f5de8e91ff1eede114e9d1fe22bda --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "98c68d418db5b73df1b05761452a845f5a35493133d7805bbed1dc89315be5c9", "size": 1167172, "path": "media/videos/eval/0_eval_0_98c68d418db5b73df1b0.mp4"}, "global_step": 1463, "_timestamp": 1737145184.8142095, "_runtime": 540.381210565567, "_step": 23, "eval/1_eval": {"_type": "video-file", "sha256": "fe99acb9f5fbe6b6c23e6a1f518986a970f45769de1d4acd955b9b5f9bb5e789", "size": 1154808, "path": "media/videos/eval/1_eval_1_fe99acb9f5fbe6b6c23e.mp4"}, "eval/num_success": 11.0, "episode": 11.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.6812262535095215, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.7194086909294128, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.8217406868934631, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.8676349520683289, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "a9b669c92c8beee6656cacd11f615b7c5aee743d336e4b927a08eddfab14d19b", "size": 1163094, "path": "media/videos/eval/2_eval_2_a9b669c92c8beee6656c.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "2988eff819b475f131997a6481c4c4ca6d5d6a9124cea83e7998596e71612259", "size": 1160636, "path": "media/videos/eval/3_eval_3_2988eff819b475f13199.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "8487a970815421048be960754b4caeed67e948d55be1d9121a1fb0628f53fb10", "size": 1171773, "path": "media/videos/eval/4_eval_4_8487a970815421048be9.mp4"}, "eval/5_eval": {"_type": "video-file", "sha256": "5eccb1be08111ca14e10f50c8b05a5c13c37a164fba6aaa66d39310d8cfd12f3", "size": 1269853, "path": "media/videos/eval/5_eval_5_5eccb1be08111ca14e10.mp4"}, "eval/6_eval": {"_type": "video-file", "sha256": "5f3fdb589a9c504e49465abf5578b1e48353445b5fb7da025d1597ab32b5a0b7", "size": 1231149, "path": "media/videos/eval/6_eval_6_5f3fdb589a9c504e4946.mp4"}, "eval/7_eval": {"_type": "video-file", "sha256": "eab2215188c302c460ba00e7f817b9602d979b290be42c792b2cff46a93ae595", "size": 1153200, "path": "media/videos/eval/7_eval_7_eab2215188c302c460ba.mp4"}, "eval/8_eval": {"_type": "video-file", "sha256": "50d0e7e81a609c529244b4bf0955c640353908a3899746c56f1df81d8ea406b8", "size": 1174619, "path": "media/videos/eval/8_eval_8_50d0e7e81a609c529244.mp4"}, "eval/9_eval": {"_type": "video-file", "sha256": "9a2233becb7986a47e63b28b21334780efa49c26963ca4e624fb1db5d670d747", "size": 1362409, "path": "media/videos/eval/9_eval_9_9a2233becb7986a47e63.mp4"}, "eval/10_eval": {"_type": "video-file", "sha256": "dc26db5ffbb288ebf69be3b37f1a149b05a67b55add127df99546db602a0c6ce", "size": 1385355, "path": "media/videos/eval/10_eval_10_dc26db5ffbb288ebf69b.mp4"}, "eval/11_eval": {"_type": "video-file", "sha256": "f83167404e8bb0859778ab785b3ec72f319922e32ba8be47628685d52a3fb45e", "size": 1451754, "path": "media/videos/eval/11_eval_11_f83167404e8bb0859778.mp4"}, "eval/video": {"_type": "video-file", "sha256": "25445a41b5bbda0ed7e2e0845eaa5c96373705c5c738fe640c1ea70a07b4176c", "size": 262, "path": "media/videos/eval/video_12_25445a41b5bbda0ed7e2.mp4"}, "_wandb": {"runtime": 2633}} \ No newline at end of file diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/logs/debug-internal.log b/h18xe90d/wandb/run-20250117_151044-h18xe90d/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..fde8e41a65f3a1b66c1349dca5ab6d5c101c296b --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/logs/debug-internal.log @@ -0,0 +1,4097 @@ +2025-01-17 15:10:44,431 INFO StreamThr :2520816 [internal.py:wandb_internal():85] W&B internal server running at pid: 2520816, started at: 2025-01-17 15:10:44.431316 +2025-01-17 15:10:44,432 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status +2025-01-17 15:10:44,448 INFO WriterThread:2520816 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/run-h18xe90d.wandb +2025-01-17 15:10:44,448 DEBUG SenderThread:2520816 [sender.py:send():379] send: header +2025-01-17 15:10:44,457 DEBUG SenderThread:2520816 [sender.py:send():379] send: run +2025-01-17 15:10:44,741 INFO SenderThread:2520816 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files +2025-01-17 15:10:44,741 INFO SenderThread:2520816 [sender.py:_start_run_threads():1188] run started: h18xe90d with start time 1737144644.432999 +2025-01-17 15:10:44,754 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: check_version +2025-01-17 15:10:44,755 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: check_version +2025-01-17 15:10:44,802 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: run_start +2025-01-17 15:10:44,811 DEBUG HandlerThread:2520816 [system_info.py:__init__():26] System info init +2025-01-17 15:10:44,811 DEBUG HandlerThread:2520816 [system_info.py:__init__():41] System info init done +2025-01-17 15:10:44,811 INFO HandlerThread:2520816 [system_monitor.py:start():194] Starting system monitor +2025-01-17 15:10:44,811 INFO SystemMonitor:2520816 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-17 15:10:44,812 INFO SystemMonitor:2520816 [interfaces.py:start():188] Started cpu monitoring +2025-01-17 15:10:44,812 INFO HandlerThread:2520816 [system_monitor.py:probe():214] Collecting system info +2025-01-17 15:10:44,813 INFO SystemMonitor:2520816 [interfaces.py:start():188] Started disk monitoring +2025-01-17 15:10:44,814 INFO SystemMonitor:2520816 [interfaces.py:start():188] Started gpu monitoring +2025-01-17 15:10:44,814 INFO SystemMonitor:2520816 [interfaces.py:start():188] Started memory monitoring +2025-01-17 15:10:44,815 INFO SystemMonitor:2520816 [interfaces.py:start():188] Started network monitoring +2025-01-17 15:10:44,824 DEBUG HandlerThread:2520816 [system_info.py:probe():152] Probing system +2025-01-17 15:10:44,826 DEBUG HandlerThread:2520816 [system_info.py:_probe_git():137] Probing git +2025-01-17 15:10:44,832 DEBUG HandlerThread:2520816 [system_info.py:_probe_git():145] Probing git done +2025-01-17 15:10:44,832 DEBUG HandlerThread:2520816 [system_info.py:probe():200] Probing system done +2025-01-17 15:10:44,832 DEBUG HandlerThread:2520816 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-17T20:10:44.824913', 'startedAt': '2025-01-17T20:10:44.420748', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3952.85125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4530.086, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 426.36322021484375}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-17 15:10:44,832 INFO HandlerThread:2520816 [system_monitor.py:probe():224] Finished collecting system info +2025-01-17 15:10:44,832 INFO HandlerThread:2520816 [system_monitor.py:probe():227] Publishing system info +2025-01-17 15:10:44,832 DEBUG HandlerThread:2520816 [system_info.py:_save_code():44] Saving code +2025-01-17 15:10:44,834 DEBUG HandlerThread:2520816 [system_info.py:_save_code():67] Saving code done +2025-01-17 15:10:44,834 DEBUG HandlerThread:2520816 [system_info.py:_save_patches():84] Saving git patches +2025-01-17 15:10:44,934 DEBUG HandlerThread:2520816 [system_info.py:_save_patches():126] Saving git patches done +2025-01-17 15:10:44,934 INFO HandlerThread:2520816 [system_monitor.py:probe():229] Finished publishing system info +2025-01-17 15:10:44,937 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:10:44,937 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-17 15:10:44,938 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-17 15:10:44,938 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-17 15:10:44,938 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-17 15:10:45,170 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: python_packages +2025-01-17 15:10:45,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:10:45,171 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: python_packages +2025-01-17 15:10:45,171 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:10:45,173 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:45,173 INFO HandlerThread:2520816 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/tb" + save: true +} +control { + relay_id: "460f5c04bdcc4fc69d0b7b404204a8cd" +} +_info { + stream_id: "h18xe90d" +} + +2025-01-17 15:10:45,173 DEBUG HandlerThread:2520816 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-17 15:10:45,201 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/0d215zcv-diff.patch +2025-01-17 15:10:45,219 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/ikt77rrz-wandb-metadata.json +2025-01-17 15:10:45,233 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/qydih3fq-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 15:10:45,255 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/be50glop-code/FISH/eval_robot.py +2025-01-17 15:10:45,310 DEBUG SenderThread:2520816 [sender.py:send():379] send: telemetry +2025-01-17 15:10:45,310 DEBUG SenderThread:2520816 [sender.py:send():379] send: tbrecord +2025-01-17 15:10:45,311 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:10:45,311 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 with policy live +2025-01-17 15:10:45,504 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/sc2eh3xv-events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:10:45,742 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:10:45,743 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/code/FISH/eval_robot.py +2025-01-17 15:10:45,743 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 15:10:45,743 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/requirements.txt +2025-01-17 15:10:45,743 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/diff.patch +2025-01-17 15:10:45,743 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-metadata.json +2025-01-17 15:10:45,743 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:10:45,743 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/code/FISH +2025-01-17 15:10:45,743 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/code +2025-01-17 15:10:46,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:47,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:47,742 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:10:48,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:49,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:50,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:10:50,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:51,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:51,874 DEBUG SenderThread:2520816 [sender.py:send():379] send: config +2025-01-17 15:10:52,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:53,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:53,743 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:10:54,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:55,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:10:55,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:55,744 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:10:56,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:57,173 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:57,745 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:10:58,173 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:10:59,173 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:00,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:11:00,171 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:11:00,173 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:00,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:01,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:01,747 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:11:02,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:03,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:04,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:05,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:06,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:06,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:07,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:08,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:09,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:10,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:11,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:11,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:12,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:13,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:14,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:15,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:11:15,171 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:11:15,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:16,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:16,307 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:16,751 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/config.yaml +2025-01-17 15:11:17,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:18,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:19,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:20,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:21,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:22,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:22,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:23,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:24,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:25,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:26,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:27,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:28,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:28,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:29,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:30,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:11:30,171 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:11:30,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:31,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:32,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:33,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:33,265 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:34,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:35,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:35,756 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:11:36,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:37,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:38,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:38,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:39,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:39,758 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:11:40,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:41,179 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:42,179 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:43,180 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:44,173 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:44,180 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:44,815 DEBUG SystemMonitor:2520816 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-17 15:11:44,816 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:11:45,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:11:45,171 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:11:45,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:46,181 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:47,181 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:48,181 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:49,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:49,245 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:50,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:51,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:52,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:53,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:54,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:55,173 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:11:55,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:56,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:57,183 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:58,183 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:11:59,183 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:00,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:12:00,171 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:12:00,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:00,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:01,183 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:02,183 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:03,183 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:04,183 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:05,017 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:12:05,017 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_98c68d418db5b73df1b0.mp4 with policy now +2025-01-17 15:12:05,017 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:12:05,018 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:12:05,019 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:12:05,020 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:12:05,184 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:05,291 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/z3xvgkha-media/videos/eval/0_eval_0_98c68d418db5b73df1b0.mp4 +2025-01-17 15:12:05,766 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:05,766 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/0_eval_0_98c68d418db5b73df1b0.mp4 +2025-01-17 15:12:05,766 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:12:05,766 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media +2025-01-17 15:12:05,766 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos +2025-01-17 15:12:05,766 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:12:06,095 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/ti2au1oi-media/videos/eval/0_eval_0_98c68d418db5b73df1b0.mp4 +2025-01-17 15:12:06,173 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:06,184 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:07,185 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:07,766 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:08,191 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:09,186 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:09,767 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:10,185 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:11,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:11,185 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:12,185 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:13,186 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:13,769 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:14,186 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:14,816 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:12:15,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:12:15,172 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:12:15,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:16,186 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:16,323 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:17,186 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:18,186 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:19,186 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:20,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:21,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:22,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:22,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:23,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:24,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:24,773 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:12:25,034 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/y26ewjg1-events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:12:25,188 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:25,773 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:26,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:27,188 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:27,729 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:27,774 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:28,188 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:29,188 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:29,774 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:30,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:12:30,172 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:12:30,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:31,188 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:31,775 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:32,188 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:33,188 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:33,609 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:33,776 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:34,188 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:35,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:35,777 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:36,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:37,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:38,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:39,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:39,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:40,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:41,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:42,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:43,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:44,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:44,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:44,817 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:12:45,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:12:45,172 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:12:45,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:46,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:47,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:47,781 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:48,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:49,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:49,271 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:50,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:51,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:52,191 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:53,191 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:54,191 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:54,421 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:12:55,191 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:55,784 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:56,192 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:56,641 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:12:56,641 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:12:56,642 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_fe99acb9f5fbe6b6c23e.mp4 with policy now +2025-01-17 15:12:56,642 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:12:56,642 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:12:56,642 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:12:56,784 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:12:56,784 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/1_eval_1_fe99acb9f5fbe6b6c23e.mp4 +2025-01-17 15:12:56,784 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:12:56,917 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/ai2sli5f-media/videos/eval/1_eval_1_fe99acb9f5fbe6b6c23e.mp4 +2025-01-17 15:12:57,193 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/fk6qtu11-media/videos/eval/1_eval_1_fe99acb9f5fbe6b6c23e.mp4 +2025-01-17 15:12:57,205 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:57,784 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:12:58,200 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:59,207 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:12:59,659 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:00,179 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:13:00,179 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:13:00,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:01,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:01,786 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:02,209 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:03,207 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:04,214 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:05,101 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:05,209 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:05,787 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:06,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:07,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:07,622 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:13:07,622 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:13:07,623 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:13:07,788 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:13:07,788 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:13:08,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:09,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:09,788 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:10,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:10,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:11,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:12,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:13,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:13,790 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:14,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:14,817 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:13:15,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:13:15,175 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:13:15,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:15,314 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:16,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:17,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:18,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:19,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:20,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:21,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:21,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:22,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:23,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:24,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:25,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:26,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:26,689 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:27,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:27,795 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:28,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:29,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:30,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:13:30,175 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:13:30,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:31,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:32,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:32,298 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:33,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:34,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:35,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:36,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:37,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:38,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:38,253 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:39,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:39,347 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:13:39,347 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a9b669c92c8beee6656c.mp4 with policy now +2025-01-17 15:13:39,351 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:13:39,352 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:13:39,352 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:13:39,352 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:13:39,609 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/m6x7gw4q-media/videos/eval/2_eval_2_a9b669c92c8beee6656c.mp4 +2025-01-17 15:13:39,799 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:39,799 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:13:39,799 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/2_eval_2_a9b669c92c8beee6656c.mp4 +2025-01-17 15:13:39,799 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:13:40,101 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/gfad54gh-media/videos/eval/2_eval_2_a9b669c92c8beee6656c.mp4 +2025-01-17 15:13:40,220 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:41,218 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:41,799 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:42,218 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:43,220 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:43,800 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:44,065 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:44,228 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:44,818 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:13:45,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:13:45,182 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:13:45,229 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:45,801 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:46,219 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:47,220 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:47,801 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:48,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:48,802 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:13:49,100 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/0g95dqdn-events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:13:49,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:49,450 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:13:49,450 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:13:49,450 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:49,450 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:13:49,802 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:49,802 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:13:50,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:51,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:51,803 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:52,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:53,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:53,804 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:54,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:54,919 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:13:55,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:56,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:57,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:57,805 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:13:58,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:13:59,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:00,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:14:00,176 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:14:00,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:00,270 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:01,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:02,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:03,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:04,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:05,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:05,651 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:05,808 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:06,223 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:07,224 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:07,809 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:08,224 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:09,224 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:09,809 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:10,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:11,180 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:11,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:11,810 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:12,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:13,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:14,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:14,818 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:14:15,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:14:15,176 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:14:15,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:16,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:16,331 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:17,226 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:18,226 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:19,226 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:20,232 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:21,116 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:14:21,116 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_2988eff819b475f13199.mp4 with policy now +2025-01-17 15:14:21,118 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:14:21,119 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:14:21,119 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:14:21,120 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:14:21,226 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:21,388 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/byrjrb8o-media/videos/eval/3_eval_3_2988eff819b475f13199.mp4 +2025-01-17 15:14:21,813 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:21,814 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:14:21,814 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/3_eval_3_2988eff819b475f13199.mp4 +2025-01-17 15:14:21,814 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:14:22,113 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/b2bazao1-media/videos/eval/3_eval_3_2988eff819b475f13199.mp4 +2025-01-17 15:14:22,155 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:22,248 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:23,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:23,814 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:24,242 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:25,252 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:25,815 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:26,244 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:27,193 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:27,245 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:28,242 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:29,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:29,816 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:30,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:14:30,190 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:14:30,245 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:30,502 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:14:30,502 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:14:30,502 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:14:30,817 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:14:30,817 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:14:31,261 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:31,817 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:32,244 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:32,261 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:33,248 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:33,818 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:34,251 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:35,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:35,819 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:36,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:37,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:38,184 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:38,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:39,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:40,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:41,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:42,244 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:43,244 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:44,184 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:44,244 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:44,818 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:14:45,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:14:45,177 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:14:45,244 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:46,244 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:47,244 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:47,823 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:48,245 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:49,245 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:49,946 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:50,245 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:51,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:51,824 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:52,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:53,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:53,825 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:54,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:54,946 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:14:55,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:55,826 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:56,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:57,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:57,827 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:14:58,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:59,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:14:59,827 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:00,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:15:00,178 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:15:00,247 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:00,295 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:01,247 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:01,828 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:02,248 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:02,504 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:15:02,504 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_8487a970815421048be9.mp4 with policy now +2025-01-17 15:15:02,509 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:15:02,509 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:15:02,510 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:15:02,510 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:15:02,773 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/avmj8hjn-media/videos/eval/4_eval_4_8487a970815421048be9.mp4 +2025-01-17 15:15:02,828 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:15:02,829 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/4_eval_4_8487a970815421048be9.mp4 +2025-01-17 15:15:02,829 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:15:03,116 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/sd8v7i4r-media/videos/eval/4_eval_4_8487a970815421048be9.mp4 +2025-01-17 15:15:03,253 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:03,829 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:04,255 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:05,261 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:06,196 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:06,272 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:07,256 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:07,830 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:08,276 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:09,256 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:09,831 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:10,284 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:11,204 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:11,276 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:11,832 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:12,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:13,280 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:14,284 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:14,819 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:15:14,833 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:15:15,138 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/xyaht6es-events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:15:15,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:15:15,179 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:15:15,285 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:15,556 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:15:15,556 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:15:15,556 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:15:15,834 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:15,834 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:15:16,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:16,781 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:17,301 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:17,835 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:18,302 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:19,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:19,836 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:20,295 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:21,302 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:21,877 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:22,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:23,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:23,837 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:24,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:25,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:26,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:27,198 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:27,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:28,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:29,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:30,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:15:30,177 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:15:30,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:31,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:32,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:32,297 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:33,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:34,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:35,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:36,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:37,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:38,198 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:38,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:39,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:40,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:41,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:42,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:43,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:44,198 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:44,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:44,819 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:15:45,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:15:45,177 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:15:45,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:46,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:47,295 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:48,295 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:49,316 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:49,472 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:49,846 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:50,018 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:15:50,018 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_5eccb1be08111ca14e10.mp4 with policy now +2025-01-17 15:15:50,020 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:15:50,021 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:15:50,021 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:15:50,021 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:15:50,308 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:50,319 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/ornukn1e-media/videos/eval/5_eval_5_5eccb1be08111ca14e10.mp4 +2025-01-17 15:15:50,847 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:15:50,847 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/5_eval_5_5eccb1be08111ca14e10.mp4 +2025-01-17 15:15:50,847 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:15:51,148 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/1jblr2x5-media/videos/eval/5_eval_5_5eccb1be08111ca14e10.mp4 +2025-01-17 15:15:51,310 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:51,847 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:52,303 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:53,312 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:54,310 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:54,745 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:15:55,317 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:55,848 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:56,309 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:57,313 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:57,849 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:58,309 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:58,849 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:15:59,312 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:15:59,608 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:15:59,609 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:15:59,609 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:15:59,850 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:15:59,850 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:15:59,850 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:15:59,905 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:00,180 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:16:00,180 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:16:00,316 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:01,316 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:01,850 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:02,330 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:03,320 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:03,851 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:04,325 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:05,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:05,322 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:05,852 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:06,322 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:07,322 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:08,322 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:09,322 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:10,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:10,324 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:11,324 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:12,324 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:13,324 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:14,324 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:14,820 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:16:15,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:16:15,178 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:16:15,260 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:15,324 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:16,325 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:17,325 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:18,325 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:19,325 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:20,325 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:21,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:21,326 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:22,326 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:23,326 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:24,326 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:25,326 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:26,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:26,326 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:27,326 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:28,326 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:29,327 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:30,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:16:30,178 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:16:30,327 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:31,284 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:31,327 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:32,338 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:33,049 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:16:33,049 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_5f3fdb589a9c504e4946.mp4 with policy now +2025-01-17 15:16:33,051 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:16:33,052 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:16:33,052 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:16:33,052 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:16:33,326 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/sdomd3mu-media/videos/eval/6_eval_6_5f3fdb589a9c504e4946.mp4 +2025-01-17 15:16:33,333 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:33,862 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:33,862 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:16:33,862 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/6_eval_6_5f3fdb589a9c504e4946.mp4 +2025-01-17 15:16:33,862 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:16:34,177 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/uaptubbe-media/videos/eval/6_eval_6_5f3fdb589a9c504e4946.mp4 +2025-01-17 15:16:34,337 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:35,335 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:35,863 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:36,359 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:36,720 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:37,344 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:37,863 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:38,349 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:39,368 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:40,383 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:41,356 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:41,865 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:42,062 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:42,356 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:42,660 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:16:42,660 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:16:42,661 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:16:42,865 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:16:42,865 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:16:43,155 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/6uc5wvic-events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:16:43,361 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:43,866 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:44,367 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:44,820 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:16:45,181 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:16:45,182 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:16:45,361 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:45,866 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:46,353 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:47,353 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:47,526 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:47,867 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:48,353 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:49,353 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:50,353 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:51,353 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:52,353 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:53,113 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:53,353 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:53,869 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:16:54,353 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:55,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:56,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:57,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:58,217 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:16:58,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:16:59,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:00,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:17:00,178 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:17:00,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:01,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:02,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:03,262 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:03,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:04,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:05,355 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:06,355 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:07,355 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:08,355 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:09,218 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:09,355 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:10,355 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:11,356 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:12,356 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:13,356 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:13,878 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:17:14,370 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:14,397 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:14,511 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:17:14,511 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_eab2215188c302c460ba.mp4 with policy now +2025-01-17 15:17:14,514 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:17:14,522 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:17:14,522 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:17:14,522 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:17:14,791 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/j00t70i9-media/videos/eval/7_eval_7_eab2215188c302c460ba.mp4 +2025-01-17 15:17:14,821 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:17:14,878 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:17:14,879 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/7_eval_7_eab2215188c302c460ba.mp4 +2025-01-17 15:17:14,879 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:17:15,173 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/7bncykqv-media/videos/eval/7_eval_7_eab2215188c302c460ba.mp4 +2025-01-17 15:17:15,195 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:17:15,196 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:17:15,398 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:15,879 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:17:16,382 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:17,397 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:18,376 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:19,407 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:19,880 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:17:20,226 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:20,396 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:21,376 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:22,397 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:23,398 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:23,882 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:17:24,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:24,712 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:17:24,712 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:17:24,713 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:17:24,882 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:17:24,882 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:17:25,236 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:25,417 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:25,882 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:17:26,414 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:27,415 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:27,883 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:17:28,433 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:29,403 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:29,884 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:17:30,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:17:30,178 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:17:30,268 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:30,403 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:31,403 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:32,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:33,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:34,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:35,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:36,230 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:36,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:37,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:38,405 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:39,405 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:40,405 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:41,231 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:41,405 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:42,405 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:43,405 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:44,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:44,821 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:17:45,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:17:45,179 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:17:45,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:46,302 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:46,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:47,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:48,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:49,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:50,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:51,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:52,231 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:52,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:53,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:54,407 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:55,437 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:56,446 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:56,692 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:17:56,692 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_50d0e7e81a609c529244.mp4 with policy now +2025-01-17 15:17:56,696 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:17:56,696 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:17:56,696 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:17:56,697 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:17:56,894 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:17:56,894 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/8_eval_8_50d0e7e81a609c529244.mp4 +2025-01-17 15:17:56,895 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:17:56,958 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/4j68napr-media/videos/eval/8_eval_8_50d0e7e81a609c529244.mp4 +2025-01-17 15:17:57,254 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/pezr5psf-media/videos/eval/8_eval_8_50d0e7e81a609c529244.mp4 +2025-01-17 15:17:57,442 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:57,754 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:17:57,895 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:17:58,449 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:59,446 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:17:59,895 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:00,206 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:18:00,206 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:18:00,445 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:01,453 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:01,896 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:02,470 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:03,427 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:03,464 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:04,459 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:05,462 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:05,898 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:06,480 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:07,463 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:08,478 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:08,766 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:18:08,766 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:18:08,766 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:08,766 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:18:08,899 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:18:08,899 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:18:09,171 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/8ey6st4m-events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:18:09,466 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:09,899 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:10,470 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:11,470 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:12,475 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:13,478 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:13,901 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:14,305 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:14,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:14,822 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:18:15,214 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:18:15,214 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:18:15,501 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:16,496 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:17,491 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:18,487 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:19,487 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:19,903 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:20,011 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:20,487 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:21,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:22,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:23,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:24,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:25,248 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:25,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:26,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:27,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:27,906 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:28,489 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:29,489 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:30,188 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:18:30,188 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:18:30,336 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:30,489 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:31,489 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:32,489 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:33,489 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:33,908 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:34,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:35,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:36,248 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:36,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:37,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:38,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:39,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:40,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:41,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:42,248 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:42,491 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:43,491 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:44,491 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:44,822 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:18:45,204 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:18:45,204 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:18:45,498 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:45,913 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:45,913 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/9_eval_9_9a2233becb7986a47e63.mp4 +2025-01-17 15:18:45,913 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:18:45,926 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:18:45,926 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_9a2233becb7986a47e63.mp4 with policy now +2025-01-17 15:18:45,930 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:18:45,931 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:18:45,931 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:18:45,931 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:18:46,179 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/bo3mrt93-media/videos/eval/9_eval_9_9a2233becb7986a47e63.mp4 +2025-01-17 15:18:46,471 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/utzcmyyr-media/videos/eval/9_eval_9_9a2233becb7986a47e63.mp4 +2025-01-17 15:18:46,529 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:46,913 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:18:47,513 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:47,914 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:47,987 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:48,520 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:49,525 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:50,526 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:51,527 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:51,915 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:52,525 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:53,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:53,657 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:53,916 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:54,533 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:55,532 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:56,530 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:56,824 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:18:56,824 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:18:56,825 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:18:56,917 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:18:56,917 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:18:57,537 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:57,918 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:18:58,563 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:59,537 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:18:59,538 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:18:59,918 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:00,216 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:19:00,216 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:19:00,533 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:01,535 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:01,919 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:02,545 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:03,541 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:04,561 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:05,268 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:05,557 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:06,539 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:07,539 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:07,921 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:08,539 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:09,539 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:10,539 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:11,266 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:11,539 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:12,539 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:13,540 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:14,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:14,823 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:19:15,195 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:19:15,195 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:19:15,541 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:16,334 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:16,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:17,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:18,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:19,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:20,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:21,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:22,266 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:22,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:23,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:24,543 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:25,543 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:26,543 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:27,267 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:27,543 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:28,543 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:29,543 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:30,196 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:19:30,196 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:19:30,543 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:31,544 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:32,348 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:32,544 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:33,562 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:33,934 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:34,553 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:34,637 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:19:34,638 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_dc26db5ffbb288ebf69b.mp4 with policy now +2025-01-17 15:19:34,681 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:19:34,681 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:19:34,682 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:19:34,682 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:19:34,924 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/g9mnvilh-media/videos/eval/10_eval_10_dc26db5ffbb288ebf69b.mp4 +2025-01-17 15:19:34,935 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:19:34,935 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/10_eval_10_dc26db5ffbb288ebf69b.mp4 +2025-01-17 15:19:34,935 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:19:35,227 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/02glaz0y-media/videos/eval/10_eval_10_dc26db5ffbb288ebf69b.mp4 +2025-01-17 15:19:35,562 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:35,935 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:36,559 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:37,584 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:38,272 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:38,576 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:39,569 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:39,937 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:40,594 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:41,569 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:42,588 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:43,529 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:43,592 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:43,938 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:44,590 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:44,823 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:19:44,885 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:19:44,885 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:19:44,885 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:19:44,939 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:19:44,939 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:19:45,201 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:19:45,201 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:19:45,243 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/a7pqmqj4-events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:19:45,624 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:45,939 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:46,620 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:47,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:47,940 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:48,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:48,913 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:49,611 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:49,940 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:50,616 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:51,617 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:52,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:53,628 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:54,271 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:19:54,623 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:55,632 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:56,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:57,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:57,944 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:19:58,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:59,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:19:59,818 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:00,196 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:20:00,196 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:20:00,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:01,616 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:02,616 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:03,616 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:04,616 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:05,274 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:05,617 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:06,617 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:07,617 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:08,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:09,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:10,274 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:10,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:11,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:12,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:13,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:14,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:14,823 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:20:15,196 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:20:15,197 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:20:15,283 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:15,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:16,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:17,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:17,952 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:20:18,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:19,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:20,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:21,275 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:21,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:22,620 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:23,620 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:24,629 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:24,955 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/11_eval_11_f83167404e8bb0859778.mp4 +2025-01-17 15:20:24,955 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:20:24,976 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:20:24,976 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_f83167404e8bb0859778.mp4 with policy now +2025-01-17 15:20:24,986 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:20:24,987 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:20:24,987 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:20:24,987 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:20:25,227 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/sm0rehjx-media/videos/eval/11_eval_11_f83167404e8bb0859778.mp4 +2025-01-17 15:20:25,554 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/gbin574l-media/videos/eval/11_eval_11_f83167404e8bb0859778.mp4 +2025-01-17 15:20:25,660 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:25,955 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:20:25,955 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:20:26,643 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:27,275 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:27,661 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:27,956 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:20:28,665 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:29,665 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:29,957 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:20:30,231 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:20:30,232 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:20:30,667 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:31,664 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:32,681 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:32,817 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:33,695 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:33,958 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:20:34,688 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:35,678 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:36,685 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:37,694 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:38,284 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:38,705 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:39,697 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:40,712 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:41,716 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:42,740 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:43,306 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:43,728 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:44,747 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:44,824 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:20:45,226 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:20:45,226 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:20:45,751 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:46,746 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:47,739 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:48,327 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:48,739 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:49,742 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:50,781 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:51,772 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:52,749 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:53,747 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:54,318 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:20:54,774 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:55,778 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:56,787 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:57,795 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:58,792 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:20:59,790 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:00,217 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:21:00,217 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:21:00,376 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:00,805 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:01,802 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:02,809 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:03,819 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:04,833 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:05,818 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:06,350 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:06,842 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:07,817 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:08,831 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:09,833 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:10,832 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:11,856 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:12,332 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:12,857 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:13,873 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:14,824 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:21:14,854 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:15,223 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:21:15,223 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:21:15,851 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:16,854 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:17,332 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:17,859 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:18,872 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:19,877 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:20,894 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:21,890 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:22,362 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:22,875 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:23,890 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:24,909 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:25,879 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:26,907 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:27,884 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:28,342 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:28,882 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:29,891 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:30,227 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:21:30,227 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:21:30,883 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:31,903 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:32,892 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:33,362 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:33,890 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:34,902 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:35,887 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:36,893 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:37,900 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:38,915 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:39,344 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:39,918 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:40,910 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:41,924 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:42,921 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:43,933 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:44,350 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:44,825 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:21:44,946 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:45,230 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:21:45,230 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:21:45,945 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:46,949 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:47,954 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:48,949 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:49,363 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:49,958 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:50,953 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:51,988 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:52,961 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:53,958 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:54,957 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:55,348 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:21:55,975 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:56,971 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:57,964 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:58,979 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:21:59,987 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:00,247 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:22:00,247 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:22:00,358 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:01,009 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:02,001 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:02,985 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:04,004 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:04,997 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:05,359 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:06,020 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:07,007 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:08,027 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:09,016 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:10,013 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:10,364 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:11,026 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:12,036 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:13,024 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:14,040 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:14,825 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:22:15,050 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:15,247 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:22:15,247 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:22:15,366 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:16,028 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:17,045 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:18,048 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:19,062 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:20,051 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:20,383 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:21,063 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:22,068 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:23,065 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:24,071 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:25,094 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:25,388 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:26,109 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:27,078 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:28,086 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:29,088 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:30,088 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:30,229 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:22:30,229 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:22:30,413 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:31,106 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:32,119 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:33,126 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:34,127 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:35,114 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:36,142 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:36,403 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:37,107 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:38,114 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:39,121 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:40,117 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:41,149 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:42,145 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:42,398 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:43,121 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:44,147 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:44,826 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:22:45,149 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:45,250 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:22:45,251 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:22:46,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:47,138 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:47,410 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:48,169 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:49,145 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:50,167 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:51,168 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:52,170 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:52,417 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:53,205 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:54,179 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:55,185 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:56,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:57,204 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:58,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:22:58,414 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:22:59,237 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:00,218 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:00,262 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:23:00,262 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:23:01,238 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:02,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:03,223 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:03,433 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:04,238 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:05,229 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:06,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:07,237 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:08,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:08,437 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:09,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:10,249 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:11,257 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:12,257 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:13,263 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:14,263 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:14,417 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:14,826 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:23:15,260 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:23:15,260 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:23:15,305 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:16,265 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:17,266 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:18,275 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:19,265 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:19,430 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:20,280 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:21,265 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:22,304 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:23,283 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:24,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:25,282 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:25,427 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:26,302 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:27,336 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:28,312 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:29,303 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:30,247 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:23:30,247 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:23:30,300 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:30,436 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:31,336 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:32,338 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:33,319 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:34,346 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:35,330 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:36,337 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:36,433 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:37,333 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:38,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:39,369 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:40,357 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:41,335 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:41,434 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:42,349 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:43,362 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:44,358 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:44,827 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:23:45,264 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:23:45,265 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:23:45,375 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:46,395 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:46,436 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:47,346 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:48,366 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:49,359 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:50,362 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:51,382 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:51,447 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:52,362 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:53,378 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:54,382 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:55,380 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:56,414 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:56,452 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:23:57,384 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:58,385 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:23:59,391 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:00,278 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:24:00,278 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:24:00,385 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:01,422 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:02,415 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:02,449 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:03,421 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:04,447 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:05,450 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:06,444 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:07,444 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:07,474 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:08,447 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:09,466 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:10,452 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:11,453 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:12,459 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:13,469 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:13,471 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:14,469 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:14,827 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:24:15,264 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:24:15,264 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:24:15,486 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:16,503 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:17,509 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:18,471 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:18,481 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:19,491 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:20,509 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:21,479 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:22,483 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:23,473 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:23,510 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:24,515 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:25,500 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:26,520 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:27,507 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:28,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:28,528 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:29,531 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:30,256 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:24:30,256 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:24:30,565 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:31,550 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:32,551 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:33,556 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:34,478 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:34,594 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:35,585 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:36,583 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:37,605 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:38,613 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:39,506 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:39,603 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:40,662 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:41,617 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:42,627 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:43,628 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:44,638 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:44,828 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:24:44,828 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:45,264 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:24:45,264 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:24:45,644 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:46,643 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:47,647 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:48,647 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:49,642 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:50,478 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:50,642 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:51,644 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:52,674 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:53,675 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:54,675 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:55,487 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:24:55,672 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:56,700 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:57,664 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:58,668 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:24:59,684 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:00,289 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:25:00,289 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:25:00,494 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:00,675 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:01,669 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:02,701 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:03,679 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:04,704 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:05,495 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:05,685 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:06,696 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:07,754 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:08,712 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:09,701 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:10,508 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:10,709 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:11,712 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:12,702 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:13,689 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:14,695 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:14,828 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:25:15,266 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:25:15,266 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:25:15,527 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:15,702 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:16,700 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:17,720 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:18,732 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:19,724 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:20,731 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:21,524 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:21,729 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:22,743 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:23,737 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:24,754 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:25,777 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:26,529 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:26,742 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:27,762 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:28,757 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:29,745 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:30,302 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:25:30,302 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:25:30,772 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:31,755 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:32,508 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:32,770 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:33,780 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:34,751 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:35,793 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:36,806 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:37,526 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:37,798 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:38,803 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:39,797 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:40,797 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:41,800 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:42,527 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:42,805 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:43,803 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:44,826 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:44,829 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:25:45,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:25:45,294 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:25:45,803 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:46,809 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:47,534 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:47,806 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:48,820 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:49,828 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:50,840 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:51,831 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:52,824 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:53,533 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:53,828 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:54,852 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:55,851 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:56,859 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:57,848 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:58,850 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:25:59,532 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:25:59,852 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:00,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:26:00,293 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:26:00,898 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:01,852 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:02,879 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:03,875 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:04,547 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:04,879 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:05,881 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:06,897 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:07,882 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:08,900 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:09,562 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:09,918 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:10,910 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:11,902 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:12,918 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:13,903 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:14,585 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:14,829 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:26:14,911 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:15,309 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:26:15,310 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:26:15,914 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:16,913 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:17,956 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:18,922 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:19,608 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:19,945 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:20,948 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:21,958 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:22,948 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:23,946 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:24,982 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:25,570 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:25,981 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:26,968 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:27,969 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:28,970 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:29,967 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:30,310 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:26:30,310 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:26:30,960 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:31,568 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:31,959 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:32,966 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:33,986 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:35,025 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:35,988 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:36,592 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:37,004 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:37,980 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:38,987 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:39,991 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:41,022 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:41,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:42,013 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:43,024 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:44,005 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:44,829 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:26:45,017 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:45,320 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:26:45,320 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:26:46,017 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:47,011 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:47,581 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:48,016 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:49,040 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:50,016 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:51,022 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:52,034 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:52,603 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:53,031 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:54,057 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:55,045 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:56,039 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:57,047 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:58,050 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:26:58,573 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:26:59,058 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:00,051 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:00,298 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:27:00,298 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:27:01,059 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:02,068 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:03,059 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:03,605 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:04,068 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:05,097 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:06,084 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:07,080 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:08,081 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:09,095 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:09,585 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:10,078 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:11,077 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:12,094 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:13,100 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:14,101 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:14,600 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:14,830 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:27:15,121 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:15,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:27:15,293 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:27:16,141 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:17,125 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:18,143 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:19,156 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:20,166 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:20,595 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:21,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:22,157 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:23,157 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:24,148 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:25,176 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:26,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:26,587 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:27,202 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:28,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:29,197 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:30,199 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:30,323 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:27:30,323 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:27:31,230 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:31,609 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:32,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:33,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:34,211 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:35,238 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:36,247 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:36,611 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:37,275 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:38,254 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:39,255 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:40,265 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:41,256 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:41,634 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:42,255 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:43,242 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:44,267 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:44,830 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:27:45,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:45,338 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:27:45,339 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:27:46,295 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:46,669 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:47,263 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:48,270 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:49,267 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:50,281 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:51,268 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:52,269 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:52,636 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:53,268 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:54,290 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:55,294 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:56,315 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:57,331 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:58,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:27:58,633 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:27:59,327 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:00,339 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:28:00,339 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:28:00,381 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:01,360 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:02,357 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:03,379 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:03,656 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:04,351 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:05,364 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:06,363 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:07,381 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:08,383 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:09,370 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:09,652 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:10,387 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:11,416 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:12,391 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:13,378 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:14,383 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:14,657 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:14,831 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:28:15,376 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:28:15,376 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:28:15,417 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:16,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:17,414 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:18,423 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:19,417 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:19,674 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:20,433 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:21,416 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:22,417 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:23,444 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:24,427 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:24,675 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:25,440 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:26,475 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:27,443 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:28,479 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:29,469 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:29,681 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:30,389 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:28:30,389 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:28:30,465 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:31,455 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:32,470 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:33,474 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:34,487 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:34,711 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:35,484 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:36,486 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:37,497 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:38,504 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:39,519 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:39,713 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:40,496 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:41,526 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:42,508 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:43,549 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:44,516 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:44,831 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:28:44,832 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:45,366 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:28:45,366 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:28:45,536 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:46,526 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:47,540 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:48,518 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:49,522 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:50,547 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:50,709 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:51,538 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:52,585 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:53,546 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:54,576 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:55,557 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:55,709 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:28:56,570 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:57,545 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:58,566 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:28:59,563 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:00,367 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:29:00,367 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:29:00,576 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:00,760 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:01,577 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:02,572 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:03,567 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:04,563 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:05,570 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:05,760 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:06,575 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:07,568 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:08,576 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:09,593 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:10,601 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:10,762 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:11,606 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:12,601 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:13,601 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:14,645 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:14,831 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:29:15,391 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:29:15,392 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:29:15,613 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:15,778 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:16,639 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:17,654 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:18,630 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:19,646 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:20,649 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:20,779 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:21,663 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:22,664 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:23,662 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:24,684 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:25,691 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:26,657 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:26,770 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:27,666 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:28,671 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:29,689 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:30,386 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:29:30,387 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:29:30,679 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:31,668 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:31,791 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:32,695 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:33,717 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:34,669 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:35,685 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:36,687 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:37,739 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:37,777 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:38,676 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:39,694 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:40,703 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:41,680 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:42,691 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:42,778 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:43,677 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:44,685 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:44,832 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:29:45,368 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:29:45,368 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:29:45,703 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:46,714 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:47,710 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:48,713 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:48,774 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:49,728 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:50,723 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:51,735 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:52,735 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:53,736 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:53,797 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:29:54,766 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:55,762 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:56,756 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:57,760 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:58,781 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:59,787 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:29:59,793 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:00,390 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:30:00,391 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:30:00,791 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:01,786 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:02,794 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:03,788 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:04,808 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:04,827 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:05,818 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:06,793 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:07,813 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:08,832 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:09,816 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:09,849 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:10,856 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:11,833 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:12,849 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:13,835 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:14,832 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:30:14,842 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:15,377 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:30:15,378 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:30:15,454 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:15,847 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:16,865 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:17,831 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:18,841 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:19,850 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:20,839 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:20,855 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:21,846 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:22,870 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:23,857 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:24,856 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:25,840 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:25,888 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:26,899 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:27,882 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:28,906 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:29,901 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:30,395 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:30:30,395 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:30:30,842 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:30,907 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:31,927 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:32,902 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:33,911 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:34,912 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:35,918 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:36,840 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:36,929 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:37,901 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:38,938 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:39,911 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:40,922 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:41,879 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:41,938 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:42,925 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:43,924 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:44,833 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:30:44,949 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:45,395 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:30:45,395 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:30:45,954 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:46,955 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:47,846 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:47,978 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:48,969 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:49,978 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:50,983 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:51,990 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:52,855 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:53,023 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:54,022 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:55,009 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:56,025 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:57,044 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:58,027 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:30:58,852 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:30:59,036 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:00,036 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:00,387 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:31:00,388 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:31:01,065 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:02,064 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:03,039 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:03,881 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:04,074 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:05,070 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:06,066 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:07,039 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:08,059 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:09,074 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:09,877 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:10,048 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:11,073 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:12,074 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:13,084 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:14,097 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:14,833 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:31:15,093 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:15,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:31:15,405 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:31:15,568 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:16,124 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:17,081 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:18,097 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:19,092 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:20,096 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:20,869 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:21,122 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:22,092 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:23,095 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:24,125 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:25,098 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:25,869 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:26,140 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:27,095 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:28,099 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:29,109 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:30,118 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:30,399 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:31:30,399 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:31:30,908 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:31,107 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:32,124 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:33,117 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:34,123 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:35,166 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:36,145 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:36,892 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:37,138 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:38,134 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:39,162 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:40,148 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:41,156 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:41,901 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:42,157 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:43,147 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:44,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:44,834 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:31:45,151 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:45,412 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:31:45,413 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:31:46,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:47,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:47,890 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:48,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:49,198 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:50,207 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:51,185 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:52,194 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:53,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:53,877 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:54,197 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:55,194 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:56,204 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:57,226 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:58,215 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:31:58,886 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:31:59,194 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:00,257 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:00,425 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:32:00,425 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:32:01,233 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:02,227 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:03,234 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:04,239 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:04,885 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:05,254 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:06,256 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:07,261 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:08,262 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:09,250 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:09,910 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:10,257 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:11,276 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:12,280 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:13,317 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:14,330 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:14,834 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:32:15,282 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:15,418 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:32:15,418 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:32:15,507 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:16,284 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:17,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:18,313 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:19,296 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:20,319 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:20,926 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:21,305 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:22,314 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:23,333 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:24,311 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:25,306 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:26,327 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:26,912 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:27,342 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:28,354 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:29,350 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:30,346 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:30,453 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:32:30,453 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:32:31,345 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:31,942 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:32,369 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:33,373 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:34,340 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:35,363 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:36,349 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:37,401 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:37,918 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:38,366 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:39,376 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:40,397 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:41,373 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:42,371 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:42,932 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:43,430 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:44,410 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:44,834 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:32:45,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:45,429 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:32:45,429 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:32:46,385 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:47,394 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:48,429 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:48,931 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:49,427 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:50,429 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:51,449 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:52,448 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:53,435 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:54,489 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:54,922 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:32:55,451 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:56,455 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:57,479 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:58,478 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:59,474 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:32:59,936 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:00,445 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:33:00,445 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:33:00,489 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:01,463 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:02,474 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:03,505 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:04,514 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:04,953 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:05,497 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:06,503 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:07,507 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:08,520 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:09,504 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:10,510 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:10,943 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:11,506 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:12,508 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:13,537 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:14,550 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:14,835 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:33:15,466 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:33:15,466 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:33:15,532 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:16,533 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:16,931 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:17,557 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:18,538 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:19,543 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:20,536 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:21,560 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:21,939 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:22,563 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:23,561 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:24,587 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:25,573 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:26,568 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:27,571 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:27,937 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:28,573 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:29,581 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:30,436 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:33:30,436 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:33:30,611 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:31,602 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:32,584 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:33,014 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:33,592 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:34,593 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:35,604 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:36,592 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:37,596 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:38,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:38,963 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:39,611 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:40,591 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:41,621 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:42,616 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:43,601 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:44,629 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:44,835 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:33:44,836 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:45,450 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:33:45,450 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:33:45,621 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:46,620 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:47,654 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:48,636 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:49,635 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:49,957 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:50,661 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:51,652 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:52,672 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:53,649 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:54,661 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:55,675 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:55,952 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:33:56,672 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:57,667 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:58,668 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:33:59,677 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:00,433 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:34:00,434 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:34:00,669 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:00,960 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:01,685 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:02,709 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:03,698 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:04,703 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:05,288 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:34:05,694 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:05,963 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:06,732 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:07,726 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:08,737 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:09,742 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:10,763 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:10,963 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:11,766 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:12,824 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:13,855 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:14,794 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:14,836 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:34:15,520 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:34:15,521 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:34:15,782 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:16,024 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:16,838 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:17,799 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:18,832 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:19,828 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:20,865 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:21,024 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:21,848 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:22,850 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:23,830 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:24,950 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:25,871 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:26,057 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:26,921 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:27,878 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:29,004 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:29,967 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:30,516 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:34:30,516 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:34:30,901 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:31,958 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:32,021 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:33,000 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:33,995 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:34,915 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:35,964 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:36,991 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:37,066 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:37,978 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:39,066 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:39,998 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:41,124 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:42,047 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:42,131 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:43,165 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:44,084 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:44,836 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:34:45,089 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:45,495 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:34:45,495 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:34:46,103 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:47,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:48,105 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:48,115 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:49,148 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:50,196 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:51,174 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:52,130 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:53,110 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:53,131 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:54,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:55,135 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:56,202 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:57,217 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:58,113 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:34:58,194 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:34:59,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:00,240 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:00,568 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:35:00,568 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:35:01,286 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:02,262 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:03,262 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:03,330 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:04,318 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:05,360 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:06,326 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:07,321 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:08,328 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:09,079 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:09,361 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:10,301 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:11,351 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:12,389 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:13,333 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:14,135 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:14,529 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:14,837 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:35:15,393 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:15,500 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:35:15,500 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:35:16,510 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:17,440 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:18,415 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:19,462 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:20,112 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:20,479 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:21,578 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:22,465 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:23,492 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:24,491 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:25,173 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:25,570 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:26,572 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:27,518 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:28,504 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:29,503 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:30,247 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:30,481 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:35:30,481 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:35:30,536 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:31,529 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:32,545 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:33,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:34,568 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:35,544 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:36,108 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:36,529 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:37,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:38,584 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:39,661 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:40,571 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:41,115 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:41,617 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:42,714 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:43,633 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:44,691 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:44,837 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:35:45,481 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:35:45,481 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:35:45,647 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:46,212 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:46,640 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:47,643 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:48,666 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:49,651 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:50,713 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:51,676 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:52,195 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:52,685 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:53,709 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:54,737 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:55,730 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:56,783 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:57,198 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:35:57,797 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:58,729 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:35:59,823 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:00,464 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:36:00,465 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:36:00,786 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:01,861 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:02,765 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:03,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:03,777 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:04,800 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:05,857 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:06,818 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:08,049 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:08,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:08,817 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:09,856 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:10,837 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:11,914 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:12,878 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:13,223 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:13,950 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:14,838 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:36:14,953 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:15,550 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:36:15,550 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:36:15,967 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:16,958 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:17,997 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:18,224 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:18,932 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:20,035 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:20,989 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:22,032 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:22,997 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:23,235 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:23,988 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:24,992 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:25,975 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:27,039 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:28,054 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:28,343 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:29,065 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:30,059 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:30,617 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:36:30,617 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:36:31,054 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:32,115 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:33,168 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:34,130 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:34,298 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:35,135 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:36,106 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:37,196 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:38,164 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:39,132 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:40,130 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:40,267 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:41,132 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:42,116 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:43,278 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:44,195 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:44,846 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:36:45,264 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:45,597 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:36:45,597 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:36:45,731 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:46,226 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:47,219 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:48,205 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:49,282 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:50,298 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:51,244 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:51,281 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:52,264 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:53,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:54,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:55,232 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:56,262 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:56,329 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:36:57,273 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:58,281 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:36:59,345 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:00,345 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:00,578 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:37:00,579 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:37:01,348 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:01,460 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:02,431 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:03,466 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:04,337 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:05,343 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:06,427 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:07,394 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:07,417 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:08,399 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:09,409 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:10,477 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:11,371 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:12,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:12,414 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:13,433 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:14,451 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:14,838 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:37:15,424 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:15,579 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:37:15,579 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:37:16,460 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:17,504 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:18,367 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:18,497 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:19,440 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:20,483 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:21,463 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:22,525 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:23,463 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:24,330 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:24,585 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:25,495 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:26,578 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:27,511 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:28,479 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:29,492 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:30,305 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:30,537 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:30,605 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:37:30,605 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:37:31,512 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:32,520 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:33,636 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:34,572 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:35,388 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:35,603 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:36,584 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:37,592 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:38,577 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:39,568 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:40,414 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:40,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:41,657 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:42,633 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:43,663 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:44,662 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:44,839 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:37:45,693 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:45,732 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:37:45,732 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:37:45,845 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:46,670 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:47,834 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:48,762 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:49,668 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:50,713 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:51,442 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:51,731 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:52,792 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:53,700 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:54,702 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:55,720 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:56,769 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:57,430 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:37:57,810 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:58,752 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:37:59,791 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:00,595 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:38:00,595 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:38:00,753 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:01,840 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:02,451 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:02,813 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:03,816 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:04,815 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:05,847 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:06,826 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:07,492 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:07,889 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:08,846 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:09,858 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:10,878 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:11,844 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:12,962 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:13,457 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:13,946 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:14,839 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:38:14,941 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:15,651 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:38:15,651 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:38:15,937 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:17,003 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:18,036 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:19,013 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:19,441 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:19,952 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:20,957 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:22,017 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:23,079 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:24,063 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:24,501 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:25,058 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:26,018 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:27,041 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:28,078 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:29,042 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:30,059 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:30,494 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:30,625 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:38:30,626 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:38:31,140 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:32,106 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:33,135 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:34,140 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:35,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:35,497 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:36,166 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:37,162 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:38,167 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:39,191 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:40,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:40,617 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:41,232 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:42,198 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:43,198 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:44,269 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:44,840 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:38:45,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:45,732 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:38:45,732 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:38:45,816 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:46,263 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:47,249 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:48,276 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:49,344 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:50,303 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:51,307 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:51,546 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:52,343 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:53,344 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:54,414 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:55,389 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:56,448 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:56,561 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:38:57,394 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:58,461 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:38:59,427 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:00,446 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:00,689 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:39:00,689 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:39:01,517 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:01,589 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:02,517 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:03,495 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:04,498 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:05,560 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:06,543 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:06,590 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:07,555 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:08,510 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:09,519 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:10,500 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:11,583 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:11,599 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:12,506 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:13,516 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:14,554 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:14,840 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:39:15,604 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:15,608 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:39:15,608 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:39:16,644 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:16,671 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:17,550 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:18,552 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:19,651 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:20,683 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:21,622 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:22,631 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:22,643 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:23,678 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:24,718 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:25,680 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:26,732 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:27,744 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:28,606 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:28,673 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:29,704 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:30,653 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:39:30,654 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:39:30,757 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:31,668 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:32,723 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:33,639 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:33,694 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:34,757 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:35,824 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:36,767 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:37,801 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:38,718 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:39,591 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:39,761 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:40,895 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:41,760 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:42,834 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:43,737 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:44,630 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:44,781 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:44,841 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:39:45,633 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:39:45,633 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:39:45,782 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:46,815 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:47,850 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:48,830 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:49,645 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:49,798 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:50,861 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:51,865 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:52,835 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:53,821 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:54,695 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:54,903 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:55,927 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:56,890 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:57,931 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:58,904 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:39:59,730 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:39:59,884 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:00,640 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:40:00,640 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:40:00,895 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:01,884 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:02,923 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:03,923 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:04,979 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:05,687 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:06,018 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:06,929 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:08,015 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:08,974 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:09,983 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:10,822 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:10,985 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:11,985 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:13,024 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:14,058 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:14,841 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:40:15,069 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:15,661 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:40:15,661 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:40:15,962 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:16,762 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:17,027 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:17,970 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:19,021 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:20,085 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:20,987 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:21,762 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:22,035 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:23,123 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:24,079 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:25,100 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:26,122 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:27,102 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:27,718 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:28,068 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:29,111 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:30,138 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:30,777 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:40:30,777 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:40:31,134 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:32,186 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:32,744 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:33,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:34,259 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:35,217 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:36,216 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:37,198 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:37,790 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:38,171 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:39,232 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:40,219 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:41,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:42,201 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:42,814 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:43,240 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:44,250 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:44,842 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:40:45,260 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:45,698 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:40:45,698 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:40:46,265 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:47,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:48,274 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:48,798 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:49,282 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:50,282 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:51,315 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:52,329 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:53,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:53,825 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:40:54,317 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:55,347 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:56,308 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:57,328 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:58,381 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:59,461 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:40:59,779 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:00,427 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:00,699 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:41:00,700 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:41:01,415 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:02,480 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:03,493 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:04,405 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:04,829 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:05,425 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:06,450 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:07,498 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:08,446 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:09,456 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:10,441 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:10,818 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:11,434 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:12,527 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:13,486 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:14,502 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:14,842 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:41:15,540 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:15,792 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:41:15,792 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:41:15,883 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:16,475 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:17,522 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:18,538 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:19,540 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:20,561 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:20,901 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:21,602 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:22,635 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:23,643 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:24,605 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:25,652 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:26,614 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:26,870 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:27,698 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:28,652 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:29,668 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:30,642 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:30,759 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:41:30,759 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:41:31,712 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:32,726 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:32,862 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:33,654 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:34,682 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:35,695 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:36,769 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:37,802 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:38,735 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:38,837 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:39,754 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:40,697 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:41,752 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:42,741 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:43,767 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:43,850 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:44,778 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:44,842 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:41:45,686 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:41:45,686 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:41:45,785 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:46,845 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:47,779 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:48,870 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:48,890 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:49,850 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:50,780 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:51,817 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:52,840 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:53,888 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:53,911 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:54,827 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:55,845 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:56,881 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:57,830 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:58,839 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:41:58,921 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:41:59,837 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:00,750 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:42:00,750 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:42:00,848 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:01,845 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:02,852 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:03,908 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:04,880 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:04,908 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:05,909 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:06,958 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:07,920 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:08,942 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:09,977 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:10,012 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:10,965 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:11,980 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:12,978 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:13,969 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:14,843 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:42:14,978 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:15,760 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:42:15,760 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:42:15,908 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:16,004 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:16,989 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:18,056 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:18,994 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:20,022 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:20,992 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:21,034 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:22,083 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:23,089 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:24,159 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:25,021 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:26,006 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:26,070 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:27,083 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:28,117 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:29,147 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:30,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:30,789 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:42:30,789 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:42:31,164 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:31,948 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:32,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:33,175 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:34,186 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:35,206 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:36,206 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:36,951 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:37,213 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:38,231 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:39,194 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:40,272 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:41,306 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:42,016 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:42,303 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:43,368 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:44,302 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:44,843 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:42:45,351 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:45,782 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:42:45,783 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:42:46,351 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:47,023 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:47,334 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:48,397 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:49,395 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:50,377 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:51,363 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:52,024 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:52,422 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:53,399 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:54,387 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:55,379 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:56,369 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:57,030 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:42:57,395 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:58,446 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:42:59,414 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:00,461 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:00,816 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:43:00,817 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:43:01,496 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:02,625 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:03,023 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:03,535 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:04,557 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:05,594 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:06,526 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:07,535 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:08,104 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:08,601 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:09,521 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:10,581 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:11,535 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:12,590 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:13,559 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:14,057 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:14,572 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:14,844 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:43:15,603 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:15,778 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:43:15,778 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:43:16,671 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:17,609 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:18,631 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:19,103 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:19,608 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:20,671 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:21,694 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:22,683 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:23,679 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:24,694 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:25,091 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:25,711 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:26,706 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:27,752 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:28,746 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:29,771 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:30,802 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:30,823 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:43:30,824 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:43:30,990 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:31,829 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:32,785 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:33,793 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:34,761 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:35,764 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:36,180 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:36,812 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:37,857 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:38,854 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:39,791 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:40,868 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:41,811 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:42,100 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:42,878 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:43,857 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:44,844 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:43:44,845 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:45,771 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:43:45,771 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:43:45,827 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:46,852 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:47,903 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:48,100 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:48,984 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:49,927 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:50,928 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:51,923 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:52,927 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:53,202 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:43:53,940 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:54,952 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:55,912 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:56,932 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:57,908 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:58,980 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:43:59,192 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:00,040 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:00,811 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:44:00,811 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:44:00,977 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:02,021 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:03,042 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:04,056 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:05,010 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:05,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:06,064 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:07,119 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:08,042 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:09,061 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:10,119 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:10,195 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:11,078 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:12,081 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:13,116 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:14,160 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:14,845 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:44:15,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:15,825 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:44:15,825 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:44:15,907 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:16,191 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:17,240 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:18,169 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:19,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:20,246 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:21,228 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:21,242 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:22,261 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:23,350 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:24,290 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:25,276 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:26,233 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:26,310 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:27,334 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:28,352 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:29,346 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:30,392 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:30,829 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:44:30,829 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:44:31,365 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:31,924 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:32,417 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:33,347 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:34,399 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:35,379 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:36,379 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:37,242 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:37,395 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:38,391 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:39,462 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:40,499 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:41,486 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:42,484 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:43,231 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:43,485 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:44,571 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:44,845 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:44:45,504 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:45,821 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:44:45,821 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:44:46,581 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:47,484 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:48,302 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:48,510 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:49,560 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:50,563 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:51,585 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:52,549 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:53,549 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:54,226 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:54,566 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:55,639 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:56,703 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:57,613 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:58,609 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:44:59,267 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:44:59,671 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:00,635 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:00,821 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:45:00,822 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:45:01,613 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:02,612 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:03,702 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:04,303 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:04,659 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:05,644 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:06,726 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:07,754 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:08,708 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:09,663 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:10,254 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:10,717 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:11,725 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:12,691 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:13,761 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:14,732 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:14,846 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:45:15,808 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:15,823 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:45:15,823 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:45:15,895 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:16,721 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:17,696 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:18,729 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:19,737 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:20,731 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:21,396 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:21,816 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:22,841 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:23,806 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:24,767 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:25,774 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:26,837 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:27,341 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:27,823 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:28,803 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:29,869 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:30,834 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:45:30,835 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:45:30,877 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:31,790 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:32,366 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:32,854 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:33,936 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:34,895 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:35,897 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:36,882 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:37,860 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:38,307 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:38,903 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:39,897 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:40,958 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:41,968 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:42,922 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:43,952 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:44,293 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:44,846 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:45:44,871 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:45,828 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:45:45,828 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:45:45,934 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:46,899 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:47,905 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:48,901 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:49,950 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:50,286 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:50,918 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:51,951 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:52,931 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:53,916 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:54,968 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:55,399 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:45:55,982 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:56,980 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:57,969 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:45:58,964 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:00,100 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:00,877 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:46:00,877 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:46:00,959 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:00,995 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:02,031 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:03,019 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:04,049 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:05,043 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:06,071 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:06,295 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:07,147 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:08,165 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:09,076 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:10,076 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:11,074 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:12,131 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:12,278 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:13,135 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:14,147 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:14,846 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:46:15,107 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:15,961 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:46:15,961 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:46:16,154 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:17,168 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:17,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:18,151 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:19,140 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:20,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:21,197 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:22,193 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:23,242 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:23,386 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:24,240 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:25,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:26,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:27,309 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:28,278 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:28,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:29,309 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:30,239 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:30,987 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:46:30,987 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:46:31,282 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:32,285 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:33,431 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:34,320 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:34,365 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:35,396 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:36,397 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:37,366 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:38,388 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:39,385 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:39,462 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:40,445 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:41,510 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:42,447 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:43,542 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:44,540 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:44,847 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:46:44,848 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:45,516 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:46,053 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:46:46,053 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:46:46,541 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:47,651 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:48,586 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:49,604 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:50,406 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:50,616 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:51,582 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:52,684 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:53,673 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:54,636 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:55,433 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:46:55,651 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:56,632 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:57,646 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:58,678 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:46:59,681 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:00,476 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:00,717 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:01,098 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:47:01,098 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:47:01,825 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:02,767 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:03,721 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:04,736 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:05,478 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:05,738 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:06,796 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:07,744 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:08,749 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:09,855 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:10,493 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:10,778 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:11,833 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:12,802 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:13,847 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:14,847 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:47:14,859 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:15,848 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:15,856 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:16,085 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:47:16,085 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:47:16,823 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:17,790 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:18,822 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:19,856 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:20,847 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:21,452 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:21,873 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:22,840 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:23,816 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:24,872 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:25,858 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:26,572 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:26,863 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:27,846 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:28,829 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:29,838 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:30,938 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:31,065 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:47:31,065 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:47:31,862 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:32,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:32,870 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:33,902 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:34,899 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:35,923 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:36,899 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:37,505 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:37,918 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:38,922 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:39,930 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:40,917 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:41,937 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:42,568 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:42,938 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:44,004 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:44,848 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:47:44,959 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:45,950 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:46,029 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:47:46,030 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:47:47,058 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:48,033 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:48,524 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:49,015 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:49,976 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:51,101 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:52,006 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:53,039 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:53,549 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:54,063 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:55,019 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:56,105 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:57,051 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:58,025 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:47:58,643 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:47:59,099 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:00,041 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:01,027 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:48:01,027 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:48:01,114 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:02,170 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:03,143 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:04,121 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:04,586 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:05,129 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:06,166 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:07,144 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:08,273 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:09,162 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:09,612 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:10,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:11,224 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:12,156 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:13,231 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:14,182 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:14,848 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:48:14,849 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:15,209 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:16,095 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:48:16,095 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:48:16,209 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:17,272 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:18,258 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:19,238 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:20,252 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:20,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:21,205 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:22,296 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:23,228 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:24,238 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:25,225 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:25,638 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:26,270 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:27,232 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:28,210 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:29,277 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:30,287 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:30,659 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:31,057 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:48:31,058 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:48:31,387 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:32,313 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:33,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:34,341 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:35,377 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:36,297 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:36,605 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:37,318 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:38,325 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:39,368 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:40,370 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:41,379 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:41,642 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:42,367 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:43,378 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:44,378 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:44,849 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:48:45,457 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:46,086 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:48:46,086 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:48:46,456 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:47,245 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:47,424 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:48,393 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:49,441 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:50,439 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:51,472 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:52,515 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:52,686 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:53,606 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:54,593 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:55,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:56,566 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:57,562 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:58,564 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:48:58,668 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:48:59,569 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:00,713 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:01,064 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:49:01,065 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:49:01,602 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:02,596 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:03,616 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:04,641 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:04,680 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:05,625 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:06,674 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:07,622 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:08,650 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:09,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:09,651 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:10,630 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:11,631 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:12,843 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:13,689 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:14,691 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:14,770 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:14,849 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:49:15,738 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:16,057 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:49:16,057 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:49:16,675 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:17,721 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:18,763 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:19,727 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:19,781 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:20,761 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:21,840 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:22,911 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:23,793 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:24,729 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:24,804 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:25,787 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:26,767 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:27,787 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:28,915 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:29,754 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:29,842 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:30,874 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:31,088 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:49:31,088 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:49:31,873 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:32,917 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:33,923 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:34,971 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:35,712 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:35,917 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:36,935 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:37,969 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:38,890 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:39,950 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:40,762 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:40,958 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:41,982 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:43,007 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:43,991 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:44,850 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:49:45,007 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:45,850 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:45,977 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:46,062 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:49:46,062 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:49:46,988 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:48,038 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:49,026 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:50,018 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:51,024 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:51,746 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:52,190 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:53,094 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:54,157 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:55,113 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:56,126 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:57,144 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:57,724 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:49:58,218 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:49:59,178 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:00,150 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:01,093 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:50:01,093 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:50:01,217 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:02,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:02,871 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:03,301 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:04,269 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:05,241 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:06,247 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:07,343 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:08,376 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:08,850 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:09,277 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:10,242 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:11,245 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:12,247 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:13,281 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:13,868 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:14,342 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:14,850 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:50:15,277 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:16,098 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:50:16,099 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:50:16,292 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:17,357 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:18,330 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:19,362 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:19,807 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:20,394 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:21,339 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:22,345 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:23,345 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:24,413 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:25,404 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:25,804 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:26,425 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:27,378 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:28,462 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:29,515 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:30,540 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:30,876 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:31,139 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:50:31,139 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:50:31,458 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:32,469 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:33,475 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:34,480 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:35,560 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:36,122 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:36,583 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:37,618 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:38,587 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:39,585 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:40,604 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:41,600 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:41,979 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:42,645 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:43,630 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:44,644 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:44,850 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:50:45,673 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:46,127 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:50:46,127 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:50:46,743 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:47,260 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:47,678 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:48,650 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:49,692 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:50,692 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:51,746 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:52,726 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:52,923 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:53,767 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:54,839 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:55,868 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:56,836 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:57,837 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:57,923 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:50:58,789 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:50:59,817 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:00,768 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:01,113 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:51:01,113 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:51:01,845 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:02,818 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:02,979 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:03,817 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:04,857 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:05,816 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:06,832 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:07,862 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:08,863 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:08,929 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:09,928 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:10,916 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:12,025 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:13,059 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:13,941 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:13,951 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:14,851 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:51:14,974 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:16,063 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:16,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:51:16,222 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:51:17,023 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:18,013 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:18,954 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:18,990 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:20,015 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:21,046 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:22,078 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:23,125 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:23,969 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:24,074 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:25,129 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:26,134 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:27,105 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:28,147 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:29,167 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:29,966 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:30,208 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:31,153 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:51:31,153 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:51:31,251 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:32,223 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:33,219 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:34,263 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:34,973 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:35,270 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:36,272 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:37,224 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:38,351 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:39,231 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:40,263 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:40,955 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:41,279 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:42,319 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:43,339 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:44,360 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:44,851 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:51:45,340 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:46,165 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:51:46,165 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:51:46,268 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:46,400 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:47,410 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:48,395 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:49,410 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:50,380 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:51,444 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:52,032 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:52,370 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:53,451 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:54,458 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:55,490 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:56,423 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:57,504 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:57,992 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:51:58,551 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:51:59,496 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:00,466 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:01,150 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:52:01,150 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:52:01,474 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:02,495 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:03,029 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:03,524 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:04,583 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:05,534 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:06,533 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:07,626 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:08,030 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:08,516 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:09,482 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:10,541 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:11,544 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:12,503 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:13,048 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:13,576 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:14,521 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:14,852 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:52:15,560 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:16,207 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:52:16,207 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:52:16,540 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:17,574 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:18,558 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:19,027 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:19,609 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:20,619 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:21,661 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:22,600 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:23,653 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:24,109 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:24,722 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:25,633 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:26,751 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:27,655 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:28,708 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:29,189 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:29,675 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:30,706 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:31,170 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:52:31,170 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:52:31,671 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:32,719 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:33,758 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:34,735 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:35,124 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:35,754 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:36,720 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:37,714 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:38,723 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:39,723 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:40,133 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:40,747 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:41,796 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:42,764 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:43,771 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:44,799 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:44,852 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:52:45,811 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:45,853 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:46,289 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:52:46,290 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:52:46,745 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:47,767 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:48,811 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:49,784 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:50,805 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:51,146 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:51,827 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:52,809 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:53,876 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:54,868 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:55,887 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:56,290 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:52:56,873 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:57,885 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:58,913 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:52:59,928 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:00,973 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:01,333 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:53:01,334 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:53:01,426 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:01,903 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:02,917 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:03,941 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:04,972 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:05,961 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:06,979 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:07,192 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:08,018 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:08,998 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:09,964 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:11,027 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:12,046 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:12,238 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:13,058 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:14,077 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:14,853 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:53:14,997 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:16,045 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:16,220 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:53:16,220 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:53:17,117 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:17,359 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:18,091 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:19,177 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:20,085 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:21,114 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:22,114 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:23,122 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:23,267 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:24,243 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:25,125 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:26,172 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:27,312 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:28,233 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:28,268 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:29,187 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:30,222 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:31,217 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:53:31,217 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:53:31,257 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:32,331 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:33,297 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:33,306 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:34,319 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:35,291 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:36,345 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:37,370 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:38,297 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:38,318 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:39,397 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:40,359 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:41,362 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:42,371 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:43,304 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:43,468 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:44,414 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:44,853 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:53:45,463 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:46,256 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:53:46,256 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:53:46,469 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:47,513 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:48,335 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:48,499 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:49,611 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:50,589 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:51,550 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:52,533 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:53,526 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:54,330 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:54,509 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:55,614 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:56,526 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:57,536 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:58,592 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:53:59,425 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:53:59,525 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:00,538 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:01,221 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:54:01,221 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:54:01,541 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:02,575 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:03,650 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:04,638 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:05,330 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:54:05,586 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:06,585 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:07,568 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:08,644 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:09,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:10,384 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:54:10,648 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:11,718 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:12,694 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:13,693 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:14,699 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:14,853 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:54:15,693 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:15,854 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:54:16,237 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:54:16,237 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:54:16,734 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:17,756 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:18,811 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:19,840 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:20,791 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:21,475 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:54:21,814 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:22,806 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:23,843 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:24,872 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:25,828 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:26,902 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:27,451 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:54:27,889 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:28,889 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:29,890 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:30,837 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:31,180 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:54:31,180 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:54:31,870 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:32,853 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:33,263 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:54:33,848 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:34,965 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:35,946 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:37,035 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:37,945 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:38,175 DEBUG SenderThread:2520816 [sender.py:send():379] send: files +2025-01-17 15:54:38,175 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file media/videos/eval/video_12_25445a41b5bbda0ed7e2.mp4 with policy now +2025-01-17 15:54:38,217 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:54:38,217 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:54:38,217 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:54:38,218 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:54:38,246 DEBUG SenderThread:2520816 [sender.py:send():379] send: telemetry +2025-01-17 15:54:38,378 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: server_info +2025-01-17 15:54:38,378 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: server_info +2025-01-17 15:54:38,428 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/3i9y75zy-media/videos/eval/video_12_25445a41b5bbda0ed7e2.mp4 +2025-01-17 15:54:38,440 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:54:38,558 DEBUG SenderThread:2520816 [sender.py:send():379] send: exit +2025-01-17 15:54:38,558 INFO SenderThread:2520816 [sender.py:send_exit():586] handling exit code: 0 +2025-01-17 15:54:38,558 INFO SenderThread:2520816 [sender.py:send_exit():588] handling runtime: 2633 +2025-01-17 15:54:38,559 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:54:38,559 INFO SenderThread:2520816 [sender.py:send_exit():594] send defer +2025-01-17 15:54:38,559 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:38,559 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-17 15:54:38,559 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:38,559 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-17 15:54:38,559 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 1 +2025-01-17 15:54:38,559 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:38,559 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-17 15:54:38,560 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:38,560 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-17 15:54:38,560 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 2 +2025-01-17 15:54:38,560 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:38,560 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-17 15:54:38,560 INFO HandlerThread:2520816 [system_monitor.py:finish():203] Stopping system monitor +2025-01-17 15:54:38,560 DEBUG SystemMonitor:2520816 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-17 15:54:38,560 INFO HandlerThread:2520816 [interfaces.py:finish():200] Joined cpu monitor +2025-01-17 15:54:38,560 DEBUG SystemMonitor:2520816 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-17 15:54:38,560 INFO HandlerThread:2520816 [interfaces.py:finish():200] Joined disk monitor +2025-01-17 15:54:38,614 INFO HandlerThread:2520816 [interfaces.py:finish():200] Joined gpu monitor +2025-01-17 15:54:38,614 INFO HandlerThread:2520816 [interfaces.py:finish():200] Joined memory monitor +2025-01-17 15:54:38,614 INFO HandlerThread:2520816 [interfaces.py:finish():200] Joined network monitor +2025-01-17 15:54:38,615 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:38,615 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-17 15:54:38,615 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 3 +2025-01-17 15:54:38,615 DEBUG SenderThread:2520816 [sender.py:send():379] send: stats +2025-01-17 15:54:38,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:38,615 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-17 15:54:38,615 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:38,615 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-17 15:54:38,615 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 4 +2025-01-17 15:54:38,615 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:38,615 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-17 15:54:38,777 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:54:38,777 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/video_12_25445a41b5bbda0ed7e2.mp4 +2025-01-17 15:54:38,778 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval +2025-01-17 15:54:39,036 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /tmp/tmp9048tr_0wandb/aciuk6x1-media/videos/eval/video_12_25445a41b5bbda0ed7e2.mp4 +2025-01-17 15:54:40,778 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:54:45,075 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:54:45,075 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:54:45,075 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: keepalive +2025-01-17 15:54:45,075 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:45,076 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-17 15:54:45,076 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 5 +2025-01-17 15:54:45,076 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:54:45,076 DEBUG SenderThread:2520816 [sender.py:send():379] send: history +2025-01-17 15:54:45,076 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:54:45,078 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:54:45,078 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:45,078 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-17 15:54:45,083 DEBUG SenderThread:2520816 [sender.py:send():379] send: summary +2025-01-17 15:54:45,083 INFO SenderThread:2520816 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:54:45,084 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:45,084 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-17 15:54:45,084 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 6 +2025-01-17 15:54:45,084 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:45,084 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-17 15:54:45,084 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:45,084 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-17 15:54:45,088 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:54:45,213 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 7 +2025-01-17 15:54:45,214 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:45,214 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-17 15:54:45,214 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:45,214 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-17 15:54:45,740 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:54:45,780 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/config.yaml +2025-01-17 15:54:45,780 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:54:47,267 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 8 +2025-01-17 15:54:47,267 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:54:47,268 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:47,268 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-17 15:54:47,268 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:47,268 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-17 15:54:47,268 INFO SenderThread:2520816 [job_builder.py:build():440] Attempting to build job artifact +2025-01-17 15:54:47,270 INFO SenderThread:2520816 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-17 15:54:47,289 INFO SenderThread:2520816 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-17 15:54:47,302 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 9 +2025-01-17 15:54:47,302 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:47,302 DEBUG SenderThread:2520816 [sender.py:send():379] send: artifact +2025-01-17 15:54:47,302 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-17 15:54:47,781 INFO Thread-12 :2520816 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:54:47,809 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:54:47,906 INFO wandb-upload_3:2520816 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzmv2qlke +2025-01-17 15:54:47,906 INFO wandb-upload_1:2520816 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpxqk64tfw +2025-01-17 15:54:48,115 INFO wandb-upload_0:2520816 [upload_job.py:push():88] Uploaded file /tmp/tmp7cvwy9fu/wandb-job.json +2025-01-17 15:54:48,756 INFO SenderThread:2520816 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ0OTIyOTkxNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ0OTEwNDY1OA==', 'versionIndex': 442}}} +2025-01-17 15:54:48,756 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:48,756 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-17 15:54:48,756 INFO SenderThread:2520816 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-17 15:54:48,782 INFO SenderThread:2520816 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files +2025-01-17 15:54:48,782 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/config.yaml config.yaml +2025-01-17 15:54:48,782 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json wandb-summary.json +2025-01-17 15:54:48,782 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 events.out.tfevents.1737144645.leonmkim-ROG-Strix-G15CS-G15CS.2520639.0 +2025-01-17 15:54:48,785 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/requirements.txt requirements.txt +2025-01-17 15:54:48,785 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-metadata.json wandb-metadata.json +2025-01-17 15:54:48,785 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 15:54:48,785 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log output.log +2025-01-17 15:54:48,786 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/diff.patch diff.patch +2025-01-17 15:54:48,788 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/video_12_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_12_25445a41b5bbda0ed7e2.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/2_eval_2_a9b669c92c8beee6656c.mp4 media/videos/eval/2_eval_2_a9b669c92c8beee6656c.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/5_eval_5_5eccb1be08111ca14e10.mp4 media/videos/eval/5_eval_5_5eccb1be08111ca14e10.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/1_eval_1_fe99acb9f5fbe6b6c23e.mp4 media/videos/eval/1_eval_1_fe99acb9f5fbe6b6c23e.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/7_eval_7_eab2215188c302c460ba.mp4 media/videos/eval/7_eval_7_eab2215188c302c460ba.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/9_eval_9_9a2233becb7986a47e63.mp4 media/videos/eval/9_eval_9_9a2233becb7986a47e63.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/6_eval_6_5f3fdb589a9c504e4946.mp4 media/videos/eval/6_eval_6_5f3fdb589a9c504e4946.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/11_eval_11_f83167404e8bb0859778.mp4 media/videos/eval/11_eval_11_f83167404e8bb0859778.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/3_eval_3_2988eff819b475f13199.mp4 media/videos/eval/3_eval_3_2988eff819b475f13199.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/10_eval_10_dc26db5ffbb288ebf69b.mp4 media/videos/eval/10_eval_10_dc26db5ffbb288ebf69b.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/8_eval_8_50d0e7e81a609c529244.mp4 media/videos/eval/8_eval_8_50d0e7e81a609c529244.mp4 +2025-01-17 15:54:48,789 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/4_eval_4_8487a970815421048be9.mp4 media/videos/eval/4_eval_4_8487a970815421048be9.mp4 +2025-01-17 15:54:48,790 INFO SenderThread:2520816 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/media/videos/eval/0_eval_0_98c68d418db5b73df1b0.mp4 media/videos/eval/0_eval_0_98c68d418db5b73df1b0.mp4 +2025-01-17 15:54:48,790 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 10 +2025-01-17 15:54:48,790 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:54:48,790 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:48,790 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-17 15:54:48,790 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:48,790 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-17 15:54:48,790 INFO SenderThread:2520816 [file_pusher.py:finish():169] shutting down file pusher +2025-01-17 15:54:48,862 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:54:48,862 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:54:48,976 INFO wandb-upload_2:2520816 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/config.yaml +2025-01-17 15:54:49,024 INFO wandb-upload_3:2520816 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/wandb-summary.json +2025-01-17 15:54:49,083 INFO wandb-upload_1:2520816 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/output.log +2025-01-17 15:54:49,086 INFO wandb-upload_0:2520816 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/files/requirements.txt +2025-01-17 15:54:49,286 INFO Thread-11 (_thread_body):2520816 [sender.py:transition_state():614] send defer: 11 +2025-01-17 15:54:49,286 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:49,286 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-17 15:54:49,287 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:49,287 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-17 15:54:49,287 INFO SenderThread:2520816 [file_pusher.py:join():175] waiting for file pusher +2025-01-17 15:54:49,289 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 12 +2025-01-17 15:54:49,289 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:49,289 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-17 15:54:49,289 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:49,289 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-17 15:54:49,289 INFO SenderThread:2520816 [file_stream.py:finish():601] file stream finish called +2025-01-17 15:54:49,488 INFO SenderThread:2520816 [file_stream.py:finish():605] file stream finish is done +2025-01-17 15:54:49,488 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 13 +2025-01-17 15:54:49,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:49,488 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-17 15:54:49,488 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:49,488 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-17 15:54:49,488 INFO SenderThread:2520816 [sender.py:transition_state():614] send defer: 14 +2025-01-17 15:54:49,488 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:54:49,488 DEBUG SenderThread:2520816 [sender.py:send():379] send: final +2025-01-17 15:54:49,488 INFO HandlerThread:2520816 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-17 15:54:49,488 DEBUG SenderThread:2520816 [sender.py:send():379] send: footer +2025-01-17 15:54:49,489 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: defer +2025-01-17 15:54:49,489 INFO SenderThread:2520816 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-17 15:54:49,602 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:54:49,602 DEBUG SenderThread:2520816 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:54:49,802 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:54:49,883 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: get_summary +2025-01-17 15:54:49,973 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-17 15:54:50,009 DEBUG HandlerThread:2520816 [handler.py:handle_request():158] handle_request: shutdown +2025-01-17 15:54:50,010 INFO HandlerThread:2520816 [handler.py:finish():882] shutting down handler +2025-01-17 15:54:50,602 INFO WriterThread:2520816 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/run-h18xe90d.wandb +2025-01-17 15:54:50,602 INFO SenderThread:2520816 [sender.py:finish():1615] shutting down sender +2025-01-17 15:54:50,602 INFO SenderThread:2520816 [file_pusher.py:finish():169] shutting down file pusher +2025-01-17 15:54:50,603 INFO SenderThread:2520816 [file_pusher.py:join():175] waiting for file pusher diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/logs/debug.log b/h18xe90d/wandb/run-20250117_151044-h18xe90d/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f16ebfd10e704f198b497772cb10ee7531c3c918 --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/logs/debug.log @@ -0,0 +1,37 @@ +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_setup.py:_flush():76] Configure stats pid to 2520639 +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/settings +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/logs/debug.log +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/wandb/run-20250117_151044-h18xe90d/logs/debug-internal.log +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_init.py:init():569] calling init triggers +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3456_0', 'wandb_notes': '3456_0_req_3921_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d'} +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_init.py:init():619] starting backend +2025-01-17 15:10:44,424 INFO MainThread:2520639 [wandb_init.py:init():623] setting up manager +2025-01-17 15:10:44,430 INFO MainThread:2520639 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-17 15:10:44,432 INFO MainThread:2520639 [wandb_init.py:init():631] backend started and connected +2025-01-17 15:10:44,447 INFO MainThread:2520639 [wandb_init.py:init():720] updated telemetry +2025-01-17 15:10:44,454 INFO MainThread:2520639 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-17 15:10:44,753 INFO MainThread:2520639 [wandb_run.py:_on_init():2435] communicating current version +2025-01-17 15:10:44,796 INFO MainThread:2520639 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-17 15:10:44,797 INFO MainThread:2520639 [wandb_init.py:init():804] starting run threads in backend +2025-01-17 15:10:45,171 INFO MainThread:2520639 [wandb_run.py:_console_start():2413] atexit reg +2025-01-17 15:10:45,171 INFO MainThread:2520639 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-17 15:10:45,171 INFO MainThread:2520639 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-17 15:10:45,172 INFO MainThread:2520639 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-17 15:10:45,173 INFO MainThread:2520639 [wandb_init.py:init():847] run started, returning control to user process +2025-01-17 15:10:45,173 INFO MainThread:2520639 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/h18xe90d/tb, True +2025-01-17 15:10:51,873 INFO MainThread:2520639 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-17 15:54:38,211 INFO MainThread:2520639 [wandb_run.py:_finish():2107] finishing run serialexperimentsleon/extrinsic_contact_downstream/h18xe90d +2025-01-17 15:54:38,263 INFO MainThread:2520639 [wandb_run.py:_atexit_cleanup():2374] got exitcode: 0 +2025-01-17 15:54:38,310 INFO MainThread:2520639 [wandb_run.py:_restore():2352] restore +2025-01-17 15:54:38,310 INFO MainThread:2520639 [wandb_run.py:_restore():2358] restore done +2025-01-17 15:54:51,019 INFO MainThread:2520639 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-17 15:54:51,043 INFO MainThread:2520639 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-17 15:54:52,124 INFO MainThread:2520639 [wandb_run.py:_footer_sync_info():3975] logging synced files diff --git a/h18xe90d/wandb/run-20250117_151044-h18xe90d/run-h18xe90d.wandb b/h18xe90d/wandb/run-20250117_151044-h18xe90d/run-h18xe90d.wandb new file mode 100644 index 0000000000000000000000000000000000000000..269431496897f4fd3c78cdc5204dee606f6a2211 --- /dev/null +++ b/h18xe90d/wandb/run-20250117_151044-h18xe90d/run-h18xe90d.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc4d18c9e9e4d0a29a24c590ccc67c879465fbcb79d5d22e996fb8dfd2501747 +size 198077 diff --git a/odxnffav/episode_rosbags/episode_0_2025-01-17-14-30-16.bag b/odxnffav/episode_rosbags/episode_0_2025-01-17-14-30-16.bag new file mode 100644 index 0000000000000000000000000000000000000000..cfd275fffc10860888f106725625fa06b2f13862 --- /dev/null +++ b/odxnffav/episode_rosbags/episode_0_2025-01-17-14-30-16.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c0f1814eb8dac57898fc25a137a8f1779a2b879637fb407c86ca8f67fd30a09 +size 1412446744 diff --git a/odxnffav/episode_rosbags/episode_12_2025-01-17-14-39-07.bag b/odxnffav/episode_rosbags/episode_12_2025-01-17-14-39-07.bag new file mode 100644 index 0000000000000000000000000000000000000000..17c3798395494146792fce1abdb590cba18364f6 --- /dev/null +++ b/odxnffav/episode_rosbags/episode_12_2025-01-17-14-39-07.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:421377c48e9d62c96811b77ed81aedcc8cc0efc8893596907ac81ece64574902 +size 1412528869 diff --git a/odxnffav/episode_rosbags/episode_14_2025-01-17-14-40-42.bag b/odxnffav/episode_rosbags/episode_14_2025-01-17-14-40-42.bag new file mode 100644 index 0000000000000000000000000000000000000000..cd43f97922d2d45a1d27ac2145d1da6ec7cd55f6 --- /dev/null +++ b/odxnffav/episode_rosbags/episode_14_2025-01-17-14-40-42.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c220fcee6b956cc180b28c809577b4a6eb5df2f952e29244247af1cec563eba4 +size 1413540107 diff --git a/odxnffav/episode_rosbags/episode_7_2025-01-17-14-35-26.bag b/odxnffav/episode_rosbags/episode_7_2025-01-17-14-35-26.bag new file mode 100644 index 0000000000000000000000000000000000000000..ac24dfc6fcbdc2dc5c092cce5acdd0580d1d1c0d --- /dev/null +++ b/odxnffav/episode_rosbags/episode_7_2025-01-17-14-35-26.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5263b8322772c9c984926ca59aabd8cd7479b853882d3b5da9df3dd25109eca +size 1413443782 diff --git a/odxnffav/episode_rosbags/episode_9_2025-01-17-14-36-55.bag b/odxnffav/episode_rosbags/episode_9_2025-01-17-14-36-55.bag new file mode 100644 index 0000000000000000000000000000000000000000..9294a6ba9c81c939df273cadcdfb0a80b49c3ca7 --- /dev/null +++ b/odxnffav/episode_rosbags/episode_9_2025-01-17-14-36-55.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea7e3a610ccb9036630afa8106bccc99b7666b6f047151c1d115c197b4f07bc4 +size 1412857451 diff --git a/roatk2bk/.hydra/config.yaml b/roatk2bk/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c6218867bdef1ad3ae0c2d6cc61cdafef42eee2 --- /dev/null +++ b/roatk2bk/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '3456_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/roatk2bk/.hydra/hydra.yaml b/roatk2bk/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97b4254669ada1364cd33ee0a95369110ce41c2c --- /dev/null +++ b/roatk2bk/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/roatk2bk/.hydra/overrides.yaml b/roatk2bk/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/roatk2bk/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/roatk2bk/episode_rosbags/aligned_depth_to_color_K.npy b/roatk2bk/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/roatk2bk/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/roatk2bk/episode_rosbags/color_K.npy b/roatk2bk/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/roatk2bk/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/roatk2bk/episode_rosbags/episode_0_2025-01-15-21-46-03.bag b/roatk2bk/episode_rosbags/episode_0_2025-01-15-21-46-03.bag new file mode 100644 index 0000000000000000000000000000000000000000..85d9ab1db78003c5e51cc7b902ecc0599d76a4e4 --- /dev/null +++ b/roatk2bk/episode_rosbags/episode_0_2025-01-15-21-46-03.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ca8d50ab03abcd2b2617a24f4ddcc1681fde24c4cde650c236038451ca240e0 +size 731243429 diff --git a/roatk2bk/episode_rosbags/episode_10_2025-01-15-21-53-06.bag b/roatk2bk/episode_rosbags/episode_10_2025-01-15-21-53-06.bag new file mode 100644 index 0000000000000000000000000000000000000000..8ebb3a8f76071cca5943e020dfb5605efd973344 --- /dev/null +++ b/roatk2bk/episode_rosbags/episode_10_2025-01-15-21-53-06.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ccc2275af9e8559b88da724b661024e93311217b645ffb0e8dc516e5851eb15 +size 833015350 diff --git a/roatk2bk/episode_rosbags/episode_11_2025-01-15-21-53-40.bag b/roatk2bk/episode_rosbags/episode_11_2025-01-15-21-53-40.bag new file mode 100644 index 0000000000000000000000000000000000000000..8d352b42201d63372ba75b87ff54f6796cf40fe2 --- /dev/null +++ b/roatk2bk/episode_rosbags/episode_11_2025-01-15-21-53-40.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94b9efc7dbef6e0c31fcea299ffb956e7228fbaa9cf0617c5d1ad7e2ac7f3232 +size 867007101 diff --git a/roatk2bk/episode_rosbags/episode_14_2025-01-15-21-56-03.bag b/roatk2bk/episode_rosbags/episode_14_2025-01-15-21-56-03.bag new file mode 100644 index 0000000000000000000000000000000000000000..45aa64586f6e928de80ae9b4e08bea8eb5b50f6e --- /dev/null +++ b/roatk2bk/episode_rosbags/episode_14_2025-01-15-21-56-03.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:663f6fffd683bd482fbef332a7fbb08754457f5d1a421cbed4c09d94122b46c1 +size 1410758297 diff --git a/roatk2bk/episode_rosbags/episode_15_2025-01-15-22-00-06.bag b/roatk2bk/episode_rosbags/episode_15_2025-01-15-22-00-06.bag new file mode 100644 index 0000000000000000000000000000000000000000..9ad6c44c4d1b134815de6f3c099f5e7a2f0c5500 --- /dev/null +++ b/roatk2bk/episode_rosbags/episode_15_2025-01-15-22-00-06.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d63ecad57ba502dbc0390fd40449b936e26450ccef2fb665994615ae9882543e +size 736791637 diff --git a/roatk2bk/episode_rosbags/episode_17_2025-01-15-22-02-19.bag b/roatk2bk/episode_rosbags/episode_17_2025-01-15-22-02-19.bag new file mode 100644 index 0000000000000000000000000000000000000000..03448b5012faaa9d7b9a0b74448b2d365d492ac0 --- /dev/null +++ b/roatk2bk/episode_rosbags/episode_17_2025-01-15-22-02-19.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c86b847b635620a765e6a61d0175d75edc3ce564503ca08a9b33968ba06cb234 +size 744947387 diff --git a/roatk2bk/episode_rosbags/episode_18_2025-01-15-22-03-02.bag b/roatk2bk/episode_rosbags/episode_18_2025-01-15-22-03-02.bag new file mode 100644 index 0000000000000000000000000000000000000000..cb848db22f499ec9dee137e5bf3dbd43c8d07c0e --- /dev/null +++ b/roatk2bk/episode_rosbags/episode_18_2025-01-15-22-03-02.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65c08b45c627b25904ec541f4b8c663cee8c6e1f0912206b1de0194e8e2cec07 +size 1409430649 diff --git a/roatk2bk/episode_rosbags/episode_19_2025-01-15-22-03-57.bag b/roatk2bk/episode_rosbags/episode_19_2025-01-15-22-03-57.bag new file mode 100644 index 0000000000000000000000000000000000000000..0e4efe87c090110e233fa257c0bc64a0734d8491 --- /dev/null +++ b/roatk2bk/episode_rosbags/episode_19_2025-01-15-22-03-57.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e72e566f03886d2a6cab8e4a5296df941acb08517e0489c753ff6f36a00aea7c +size 1414582160 diff --git a/roatk2bk/episode_rosbags/episode_2_2025-01-15-21-47-30.bag b/roatk2bk/episode_rosbags/episode_2_2025-01-15-21-47-30.bag new file mode 100644 index 0000000000000000000000000000000000000000..c1aac5291b7da32f4aa38bcb58a86ae380082a8c --- /dev/null +++ b/roatk2bk/episode_rosbags/episode_2_2025-01-15-21-47-30.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38249d7d56c5761efc5f90e3c73d2a3407638323d3cb33471f6ee9858c3baccd +size 839405534 diff --git a/roatk2bk/episode_rosbags/episode_4_2025-01-15-21-48-45.bag b/roatk2bk/episode_rosbags/episode_4_2025-01-15-21-48-45.bag new file mode 100644 index 0000000000000000000000000000000000000000..cf70d813ac8c40db6d3bca8efbe70c918b2343a6 --- /dev/null +++ b/roatk2bk/episode_rosbags/episode_4_2025-01-15-21-48-45.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:300ad5275d10879051892aade692869281e31a0453032f0a4e6a668ae55100cf +size 852073970 diff --git a/roatk2bk/episode_rosbags/episode_5_2025-01-15-21-49-16.bag b/roatk2bk/episode_rosbags/episode_5_2025-01-15-21-49-16.bag new file mode 100644 index 0000000000000000000000000000000000000000..29fbd56b33d5b4b6f01125bce5ed131ba65357d6 --- /dev/null +++ b/roatk2bk/episode_rosbags/episode_5_2025-01-15-21-49-16.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d2e191980615d8dc6f7b59c7f63ba04f203291ccdb97f700954939692d6a40b +size 1411999434 diff --git a/roatk2bk/episode_rosbags/episode_8_2025-01-15-21-51-45.bag b/roatk2bk/episode_rosbags/episode_8_2025-01-15-21-51-45.bag new file mode 100644 index 0000000000000000000000000000000000000000..406b49c29918d07f41fa66bf198307bec947bb20 --- /dev/null +++ b/roatk2bk/episode_rosbags/episode_8_2025-01-15-21-51-45.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c31144e64d4955be5c5929aa588371de5ec26c9fe3b0c43d3afdb7e09c571881 +size 1406755557 diff --git a/roatk2bk/episode_rosbags/episode_9_2025-01-15-21-52-35.bag b/roatk2bk/episode_rosbags/episode_9_2025-01-15-21-52-35.bag new file mode 100644 index 0000000000000000000000000000000000000000..055634befebd7a4b00f07e8f2e9795232fb47307 --- /dev/null +++ b/roatk2bk/episode_rosbags/episode_9_2025-01-15-21-52-35.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c28adfbe42b276e42b25cf6aaa1b0db70af82bd5896fcbf8014339f5b22d62c +size 840731905 diff --git a/roatk2bk/eval_video/0_eval.mp4 b/roatk2bk/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4e04c623aff9ed8198593d50fcc11f61c91475be --- /dev/null +++ b/roatk2bk/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2eb6cf19687a4fabfe3c6f8e612266c1da2cc0076d60b65057514a926a74144f +size 660308 diff --git a/roatk2bk/eval_video/10_eval.mp4 b/roatk2bk/eval_video/10_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c44c0857412c3888597fe9729489631f824fef7e --- /dev/null +++ b/roatk2bk/eval_video/10_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e43fe2801f3a905e582de81a9680045e1a4067f03042a5ff19f96803aa6f698 +size 1031684 diff --git a/roatk2bk/eval_video/11_eval.mp4 b/roatk2bk/eval_video/11_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..82ee335b004ac11441bdf08b4e71f471bf38ec75 --- /dev/null +++ b/roatk2bk/eval_video/11_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4263c7dc48b07a1c552cc54d84f88093f2e788ee860e0a2d14e44c2cc1fdaeef +size 1084332 diff --git a/roatk2bk/eval_video/12_eval.mp4 b/roatk2bk/eval_video/12_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6e4c28c10969c11aef3116208dd2ee0b3770d95f --- /dev/null +++ b/roatk2bk/eval_video/12_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:241be90b3f1c54884e79ee6f66e49e2b9cd981f6a46f20431ce6ffa92227c5b2 +size 1167206 diff --git a/roatk2bk/eval_video/13_eval.mp4 b/roatk2bk/eval_video/13_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7505f4a66141c528032949a427cc5bfbe03ece02 --- /dev/null +++ b/roatk2bk/eval_video/13_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb3449f02cfc7cf55b8d5c8b2858c5450855681d80b52de97d006761f8ae751b +size 1419465 diff --git a/roatk2bk/eval_video/14_eval.mp4 b/roatk2bk/eval_video/14_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f9062562c7d4ae9e368c27df289d0faf1528f9bb --- /dev/null +++ b/roatk2bk/eval_video/14_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:074bdfc80a4a9d765f986d8266f086e36f6336cbfc94d03d20f82b5898467960 +size 1166864 diff --git a/roatk2bk/eval_video/15_eval.mp4 b/roatk2bk/eval_video/15_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0be7b49ee55c4e0d9cab9ef43d8fde613a9c5e87 --- /dev/null +++ b/roatk2bk/eval_video/15_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76fa4e81e9e73fef9cf0cbf0cae5de5fadd030e8577b710e5fbbc635395fbbad +size 961267 diff --git a/roatk2bk/eval_video/16_eval.mp4 b/roatk2bk/eval_video/16_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..35878cc45c89c73ccaf9e56f66bf614e34393ad0 --- /dev/null +++ b/roatk2bk/eval_video/16_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5019757f76965e10782b8dbf76cb0ec5dd9466a48a2c73f37eaae45a01df3902 +size 1505807 diff --git a/roatk2bk/eval_video/17_eval.mp4 b/roatk2bk/eval_video/17_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..415050cdcbaa3f4f78d6645b8f7e282c5e7e233d --- /dev/null +++ b/roatk2bk/eval_video/17_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f96b121fd6f99c60674db7dbf14916a988cd7103feb371d1024b26d7d81e965 +size 680421 diff --git a/roatk2bk/eval_video/18_eval.mp4 b/roatk2bk/eval_video/18_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..85b9b2104bf48e99768a588bc493e0090c07dd93 --- /dev/null +++ b/roatk2bk/eval_video/18_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8c84019c8dc8ba0e4ead689259df8c2ee18d6119c191be15ecdb4cd11fc2f0f +size 1441508 diff --git a/roatk2bk/eval_video/19_eval.mp4 b/roatk2bk/eval_video/19_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a8251c6e3499a0a662f47571fb875853fef7ac5b --- /dev/null +++ b/roatk2bk/eval_video/19_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f737e19fed74dd298b263bf46de6d1133a6a7db350c8d12a1fdd73af013c15b +size 1623338 diff --git a/roatk2bk/eval_video/1_eval.mp4 b/roatk2bk/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b7b4fbb505f3c7a6024f46f7e3eb58f3fc9abc15 --- /dev/null +++ b/roatk2bk/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5d7870b7b04a6ef485f99d3b1ba052460f021a02fc0a4bcb269c4a104c53731 +size 1158221 diff --git a/roatk2bk/eval_video/2_eval.mp4 b/roatk2bk/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..181e6cebd1809a6788d1392e25ffdddd9d6b2a58 --- /dev/null +++ b/roatk2bk/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f98c31cd2e05324cffd88658a9b1b179814eff0305d38aba54e59921246de742 +size 810213 diff --git a/roatk2bk/eval_video/3_eval.mp4 b/roatk2bk/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..de47f098ac8dfb66520fa77a28e642d4ee0c52c4 --- /dev/null +++ b/roatk2bk/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89821fb09b372630838b8abd67ec78c34acf566da143b01acc2d08888bb0bc6c +size 1153488 diff --git a/roatk2bk/eval_video/4_eval.mp4 b/roatk2bk/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..be7d633e95c3b15759f18219f17205e3d270d114 --- /dev/null +++ b/roatk2bk/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fb52356d0a213033cfe25b61a1934cbe1ee32f73cd0b718905313f2644136c0 +size 897581 diff --git a/roatk2bk/eval_video/5_eval.mp4 b/roatk2bk/eval_video/5_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f9ea809294c8115d5a59122311bd41632ea879f0 --- /dev/null +++ b/roatk2bk/eval_video/5_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a69fcdfd49e2066116b64ed866b1feec0677babe9575a14c121d0519d8c7ddc6 +size 1233195 diff --git a/roatk2bk/eval_video/6_eval.mp4 b/roatk2bk/eval_video/6_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9aa2e666c5ee3808df8759634fa5d7268ebfb951 --- /dev/null +++ b/roatk2bk/eval_video/6_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae418a6c99a51b4201d6c6588f7cb94c4106062b3f5a598d62fb3d4c361fe692 +size 1234700 diff --git a/roatk2bk/eval_video/7_eval.mp4 b/roatk2bk/eval_video/7_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e92cdd03054010ee56db87f7277e6eeea20c0b79 --- /dev/null +++ b/roatk2bk/eval_video/7_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f02cea2dbceefebb3b97fe2cabe09bff692847bbfd980a30122dc91aeeff2e88 +size 1252184 diff --git a/roatk2bk/eval_video/8_eval.mp4 b/roatk2bk/eval_video/8_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9b8e2a4591a0ad2fa9ad824ea5d462f970f62dac --- /dev/null +++ b/roatk2bk/eval_video/8_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb8f14c1c353cc80983ac758cdb4a327ec227033c4ac10db233eaab9871e1ff0 +size 1248692 diff --git a/roatk2bk/eval_video/9_eval.mp4 b/roatk2bk/eval_video/9_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e27fb42dbe3ba64595aa8590aa9222196db401b9 --- /dev/null +++ b/roatk2bk/eval_video/9_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19795df8831a10c108372bc2039f39d52ab4951b1f2751808168cafd03ed2c78 +size 835686 diff --git a/roatk2bk/tb/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 b/roatk2bk/tb/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 new file mode 100644 index 0000000000000000000000000000000000000000..b5259e3d71e14e2d8232e603100fccc5533bf95f --- /dev/null +++ b/roatk2bk/tb/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69513180926b5bc9c1c9f95a8538fb26d2be6bf61e66970be4898d4a3be3f9a1 +size 22764 diff --git a/roatk2bk/wandb/debug-internal.log b/roatk2bk/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3a88dba77f6e2088adb58db153e8ffea84ee5770 --- /dev/null +++ b/roatk2bk/wandb/debug-internal.log @@ -0,0 +1,2430 @@ +2025-01-15 21:45:12,624 INFO StreamThr :2470873 [internal.py:wandb_internal():85] W&B internal server running at pid: 2470873, started at: 2025-01-15 21:45:12.624488 +2025-01-15 21:45:12,625 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status +2025-01-15 21:45:12,637 INFO WriterThread:2470873 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/run-roatk2bk.wandb +2025-01-15 21:45:12,638 DEBUG SenderThread:2470873 [sender.py:send():379] send: header +2025-01-15 21:45:12,645 DEBUG SenderThread:2470873 [sender.py:send():379] send: run +2025-01-15 21:45:12,893 INFO SenderThread:2470873 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files +2025-01-15 21:45:12,893 INFO SenderThread:2470873 [sender.py:_start_run_threads():1188] run started: roatk2bk with start time 1736995512.624682 +2025-01-15 21:45:12,912 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: check_version +2025-01-15 21:45:12,912 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: check_version +2025-01-15 21:45:12,986 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: run_start +2025-01-15 21:45:12,990 DEBUG HandlerThread:2470873 [system_info.py:__init__():26] System info init +2025-01-15 21:45:12,991 DEBUG HandlerThread:2470873 [system_info.py:__init__():41] System info init done +2025-01-15 21:45:12,991 INFO HandlerThread:2470873 [system_monitor.py:start():194] Starting system monitor +2025-01-15 21:45:12,991 INFO SystemMonitor:2470873 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-15 21:45:12,991 INFO HandlerThread:2470873 [system_monitor.py:probe():214] Collecting system info +2025-01-15 21:45:12,991 INFO SystemMonitor:2470873 [interfaces.py:start():188] Started cpu monitoring +2025-01-15 21:45:12,992 INFO SystemMonitor:2470873 [interfaces.py:start():188] Started disk monitoring +2025-01-15 21:45:12,992 INFO SystemMonitor:2470873 [interfaces.py:start():188] Started gpu monitoring +2025-01-15 21:45:12,993 INFO SystemMonitor:2470873 [interfaces.py:start():188] Started memory monitoring +2025-01-15 21:45:12,993 INFO SystemMonitor:2470873 [interfaces.py:start():188] Started network monitoring +2025-01-15 21:45:13,004 DEBUG HandlerThread:2470873 [system_info.py:probe():152] Probing system +2025-01-15 21:45:13,005 DEBUG HandlerThread:2470873 [system_info.py:_probe_git():137] Probing git +2025-01-15 21:45:13,010 DEBUG HandlerThread:2470873 [system_info.py:_probe_git():145] Probing git done +2025-01-15 21:45:13,010 DEBUG HandlerThread:2470873 [system_info.py:probe():200] Probing system done +2025-01-15 21:45:13,010 DEBUG HandlerThread:2470873 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-16T02:45:13.004292', 'startedAt': '2025-01-16T02:45:12.616933', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.5323750000002, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.919, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 326.04469299316406}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-15 21:45:13,010 INFO HandlerThread:2470873 [system_monitor.py:probe():224] Finished collecting system info +2025-01-15 21:45:13,010 INFO HandlerThread:2470873 [system_monitor.py:probe():227] Publishing system info +2025-01-15 21:45:13,010 DEBUG HandlerThread:2470873 [system_info.py:_save_code():44] Saving code +2025-01-15 21:45:13,011 DEBUG HandlerThread:2470873 [system_info.py:_save_code():67] Saving code done +2025-01-15 21:45:13,011 DEBUG HandlerThread:2470873 [system_info.py:_save_patches():84] Saving git patches +2025-01-15 21:45:13,109 DEBUG HandlerThread:2470873 [system_info.py:_save_patches():126] Saving git patches done +2025-01-15 21:45:13,109 INFO HandlerThread:2470873 [system_monitor.py:probe():229] Finished publishing system info +2025-01-15 21:45:13,113 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:45:13,113 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-15 21:45:13,113 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-15 21:45:13,113 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-15 21:45:13,113 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-15 21:45:13,359 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: python_packages +2025-01-15 21:45:13,359 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: python_packages +2025-01-15 21:45:13,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:45:13,360 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:45:13,363 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:13,363 INFO HandlerThread:2470873 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/tb" + save: true +} +control { + relay_id: "7e40abce871040128ee2222932bf4fd5" +} +_info { + stream_id: "roatk2bk" +} + +2025-01-15 21:45:13,364 DEBUG HandlerThread:2470873 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-15 21:45:13,367 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/2krs8t93-wandb-metadata.json +2025-01-15 21:45:13,395 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/i5dtzjt2-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-15 21:45:13,398 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/5i0rcxno-diff.patch +2025-01-15 21:45:13,404 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/zzqthulp-code/FISH/eval_robot.py +2025-01-15 21:45:13,535 DEBUG SenderThread:2470873 [sender.py:send():379] send: telemetry +2025-01-15 21:45:13,536 DEBUG SenderThread:2470873 [sender.py:send():379] send: tbrecord +2025-01-15 21:45:13,536 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:45:13,536 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 with policy live +2025-01-15 21:45:13,712 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/2kz91b1o-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:45:13,895 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:45:13,895 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-15 21:45:13,895 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/requirements.txt +2025-01-15 21:45:13,895 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/diff.patch +2025-01-15 21:45:13,895 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:45:13,895 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-metadata.json +2025-01-15 21:45:13,895 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/code/FISH/eval_robot.py +2025-01-15 21:45:13,895 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/code/FISH +2025-01-15 21:45:13,896 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/code +2025-01-15 21:45:14,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:15,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:15,894 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:45:16,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:17,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:18,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:45:18,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:19,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:20,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:21,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:22,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:23,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:45:23,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:24,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:25,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:26,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:27,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:28,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:45:28,360 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:45:28,405 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:28,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:45:29,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:30,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:31,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:32,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:33,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:34,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:45:34,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:35,363 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:36,363 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:37,363 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:38,363 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:39,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:45:39,363 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:40,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:41,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:41,678 DEBUG SenderThread:2470873 [sender.py:send():379] send: config +2025-01-15 21:45:42,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:43,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:45:43,360 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:45:43,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:43,901 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:45:44,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:44,435 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:45:44,902 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/config.yaml +2025-01-15 21:45:45,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:46,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:47,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:47,903 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:45:48,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:49,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:50,210 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:45:50,365 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:51,365 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:51,904 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:45:52,365 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:53,365 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:54,365 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:55,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:45:55,365 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:56,365 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:57,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:58,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:45:58,360 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:45:58,405 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:59,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:00,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:00,524 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:01,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:01,907 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:46:02,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:03,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:04,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:05,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:05,527 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:05,908 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:46:06,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:07,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:08,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:09,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:10,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:11,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:11,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:12,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:12,993 DEBUG SystemMonitor:2470873 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-15 21:46:12,994 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:46:13,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:46:13,360 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:46:13,401 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:14,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:15,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:16,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:16,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:17,368 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:18,368 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:18,469 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:46:18,470 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:46:18,470 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_2eb6cf19687a4fabfe3c.mp4 with policy now +2025-01-15 21:46:18,470 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:46:18,470 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:46:18,471 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:46:18,768 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/z7jz6cz9-media/videos/eval/0_eval_0_2eb6cf19687a4fabfe3c.mp4 +2025-01-15 21:46:18,912 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/0_eval_0_2eb6cf19687a4fabfe3c.mp4 +2025-01-15 21:46:18,912 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:46:18,912 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media +2025-01-15 21:46:18,912 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos +2025-01-15 21:46:18,912 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:46:19,189 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/o7263fbs-media/videos/eval/0_eval_0_2eb6cf19687a4fabfe3c.mp4 +2025-01-15 21:46:19,369 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:19,912 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:46:20,368 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:21,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:21,954 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:22,369 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:23,369 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:23,914 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:46:24,369 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:25,369 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:25,914 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:46:26,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:27,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:27,706 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:28,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:46:28,360 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:46:28,401 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:29,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:30,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:31,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:32,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:33,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:33,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:34,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:35,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:36,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:37,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:38,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:38,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:38,920 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:46:39,179 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/6brbwr5a-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:46:39,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:39,920 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:46:40,372 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:41,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:41,921 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:46:42,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:43,003 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:46:43,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:46:43,361 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:46:43,372 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:43,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:43,921 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:46:44,372 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:45,372 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:46,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:47,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:48,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:49,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:49,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:50,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:51,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:52,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:53,374 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:54,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:54,374 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:55,374 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:56,374 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:57,374 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:58,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:46:58,361 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:46:58,405 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:59,374 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:59,420 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:00,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:01,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:02,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:03,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:04,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:05,363 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:05,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:06,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:07,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:08,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:09,376 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:10,376 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:10,404 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:10,445 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:47:10,446 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:47:10,446 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b5d7870b7b04a6ef485f.mp4 with policy now +2025-01-15 21:47:10,446 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:47:10,446 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:47:10,447 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:47:10,708 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/bgeeyww1-media/videos/eval/1_eval_1_b5d7870b7b04a6ef485f.mp4 +2025-01-15 21:47:10,930 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:47:10,930 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/1_eval_1_b5d7870b7b04a6ef485f.mp4 +2025-01-15 21:47:10,930 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:47:11,202 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/kgz7uqes-media/videos/eval/1_eval_1_b5d7870b7b04a6ef485f.mp4 +2025-01-15 21:47:11,377 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:11,930 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:12,377 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:12,995 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:47:13,368 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:47:13,368 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:47:13,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:14,380 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:15,385 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:15,932 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:16,093 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:16,385 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:17,381 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:17,932 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:18,380 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:19,380 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:20,381 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:21,369 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:21,381 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:22,381 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:23,381 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:24,381 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:25,381 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:26,381 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:26,588 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:26,936 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:47:27,382 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:27,810 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:47:27,811 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:47:27,811 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:47:27,936 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:27,936 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:47:28,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:47:28,363 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:47:28,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:29,382 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:29,937 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:30,382 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:31,382 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:31,723 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:31,937 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:32,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:33,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:34,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:35,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:36,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:37,368 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:37,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:38,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:39,384 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:40,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:41,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:42,369 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:42,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:42,995 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:47:43,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:47:43,362 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:47:43,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:44,384 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:45,384 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:46,384 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:47,386 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:47,478 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:47,943 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:48,113 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:47:48,113 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_f98c31cd2e05324cffd8.mp4 with policy now +2025-01-15 21:47:48,113 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:47:48,114 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:47:48,114 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:47:48,115 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:47:48,354 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/22gfzdwj-media/videos/eval/2_eval_2_f98c31cd2e05324cffd8.mp4 +2025-01-15 21:47:48,389 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:48,944 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:47:48,944 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/2_eval_2_f98c31cd2e05324cffd8.mp4 +2025-01-15 21:47:48,944 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:47:49,217 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/yjg1aet9-media/videos/eval/2_eval_2_f98c31cd2e05324cffd8.mp4 +2025-01-15 21:47:49,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:49,944 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:50,389 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:50,945 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:51,390 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:52,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:52,793 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:53,397 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:53,946 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:54,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:55,393 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:55,947 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:47:55,947 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:56,200 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/mlrje92h-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:47:56,388 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:56,617 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:47:56,617 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:47:56,617 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:47:56,947 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:47:57,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:57,905 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:57,947 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:58,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:47:58,367 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:47:58,413 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:59,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:00,390 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:01,390 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:01,949 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:48:02,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:03,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:03,390 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:04,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:05,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:06,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:07,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:08,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:08,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:09,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:10,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:11,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:12,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:12,996 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:48:13,363 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:48:13,363 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:48:13,405 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:13,485 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:14,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:15,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:16,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:17,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:18,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:19,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:19,393 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:20,393 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:21,393 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:22,393 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:23,393 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:24,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:24,393 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:25,393 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:26,393 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:27,395 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:27,958 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:48:27,996 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:48:27,996 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_89821fb09b372630838b.mp4 with policy now +2025-01-15 21:48:28,005 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:48:28,006 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:48:28,006 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:48:28,006 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:48:28,254 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/b0rvfr7r-media/videos/eval/3_eval_3_89821fb09b372630838b.mp4 +2025-01-15 21:48:28,363 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:48:28,363 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:48:28,394 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:28,958 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:48:28,959 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/3_eval_3_89821fb09b372630838b.mp4 +2025-01-15 21:48:28,959 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:48:29,255 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/rjnbpuoi-media/videos/eval/3_eval_3_89821fb09b372630838b.mp4 +2025-01-15 21:48:29,398 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:29,419 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:29,959 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:48:30,403 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:30,959 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:48:31,403 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:32,403 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:33,427 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:33,960 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:48:34,403 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:34,457 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:35,405 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:35,961 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:48:36,402 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:37,405 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:38,411 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:39,404 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:40,382 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:40,420 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:41,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:41,672 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:48:41,672 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:48:41,673 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:48:41,963 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:48:41,963 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:48:42,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:42,996 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:48:43,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:48:43,367 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:48:43,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:43,964 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:48:44,410 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:45,411 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:45,964 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:48:46,313 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:46,416 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:47,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:47,965 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:48:48,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:49,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:50,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:51,372 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:51,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:52,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:53,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:54,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:55,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:56,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:56,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:57,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:58,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:48:58,364 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:48:58,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:59,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:00,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:01,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:01,442 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:02,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:03,416 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:03,971 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:04,245 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:49:04,245 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_9fb52356d0a213033cfe.mp4 with policy now +2025-01-15 21:49:04,254 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:49:04,255 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:49:04,255 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:49:04,255 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:49:04,411 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:04,483 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/6v0b62sc-media/videos/eval/4_eval_4_9fb52356d0a213033cfe.mp4 +2025-01-15 21:49:04,971 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:49:04,971 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/4_eval_4_9fb52356d0a213033cfe.mp4 +2025-01-15 21:49:04,971 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:49:05,243 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/9pexg8i7-media/videos/eval/4_eval_4_9fb52356d0a213033cfe.mp4 +2025-01-15 21:49:05,416 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:05,971 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:06,411 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:07,286 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:07,419 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:08,422 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:09,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:09,973 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:10,422 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:11,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:11,974 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:12,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:12,737 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:12,997 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:49:13,369 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:49:13,369 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:49:13,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:13,713 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:49:13,713 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:49:13,713 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:49:13,974 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:49:13,975 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:13,975 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:49:14,254 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/3wz60emi-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:49:14,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:15,421 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:15,975 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:16,421 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:17,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:17,976 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:18,005 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:18,435 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:19,421 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:19,977 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:20,421 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:21,421 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:22,421 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:23,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:23,421 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:24,421 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:25,422 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:26,422 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:27,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:28,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:49:28,365 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:49:28,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:28,428 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:29,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:30,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:31,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:32,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:33,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:34,376 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:34,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:35,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:36,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:37,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:38,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:39,376 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:39,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:40,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:41,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:42,425 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:42,997 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:49:43,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:49:43,365 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:49:43,425 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:44,425 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:44,510 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:45,425 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:46,435 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:47,038 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:49:47,038 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_a69fcdfd49e2066116b6.mp4 with policy now +2025-01-15 21:49:47,042 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:49:47,042 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:49:47,042 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:49:47,042 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:49:47,335 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/mhfqktbw-media/videos/eval/5_eval_5_a69fcdfd49e2066116b6.mp4 +2025-01-15 21:49:47,429 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:47,987 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:47,987 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:49:47,987 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/5_eval_5_a69fcdfd49e2066116b6.mp4 +2025-01-15 21:49:47,987 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:49:48,334 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/xj0rb057-media/videos/eval/5_eval_5_a69fcdfd49e2066116b6.mp4 +2025-01-15 21:49:48,441 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:48,987 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:49,440 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:50,405 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:50,439 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:51,439 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:51,988 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:52,438 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:53,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:54,440 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:55,445 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:55,617 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:55,990 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:56,442 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:57,445 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:58,378 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:49:58,379 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:49:58,449 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:59,444 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:00,453 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:00,771 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:50:00,771 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:50:00,772 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:00,772 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:50:00,991 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:50:00,992 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:50:01,444 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:01,992 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:02,445 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:03,446 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:03,992 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:04,446 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:05,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:05,993 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:06,382 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:06,446 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:07,446 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:07,994 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:08,446 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:09,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:10,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:11,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:12,379 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:12,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:12,998 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:50:13,365 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:50:13,365 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:50:13,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:14,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:15,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:16,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:17,444 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:17,448 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:18,449 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:19,449 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:20,449 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:21,449 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:22,450 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:22,987 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:23,450 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:24,000 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:24,450 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:25,451 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:26,451 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:27,451 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:28,365 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:50:28,365 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:50:28,451 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:28,504 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:29,451 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:30,451 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:31,451 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:32,451 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:33,453 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:34,004 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:34,149 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:34,435 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:50:34,435 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_ae418a6c99a51b4201d6.mp4 with policy now +2025-01-15 21:50:34,439 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:50:34,439 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:50:34,439 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:50:34,440 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:50:34,461 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:34,690 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/56yq0pm1-media/videos/eval/6_eval_6_ae418a6c99a51b4201d6.mp4 +2025-01-15 21:50:35,004 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:50:35,004 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/6_eval_6_ae418a6c99a51b4201d6.mp4 +2025-01-15 21:50:35,004 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:50:35,285 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/i8416a62-media/videos/eval/6_eval_6_ae418a6c99a51b4201d6.mp4 +2025-01-15 21:50:35,462 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:36,004 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:36,463 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:37,461 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:38,468 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:39,398 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:39,481 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:40,006 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:40,483 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:41,466 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:42,007 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:42,469 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:42,998 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:50:43,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:50:43,370 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:50:43,485 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:44,445 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:44,472 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:45,493 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:46,485 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:47,489 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:48,488 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:49,491 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:50,398 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:50,489 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:51,494 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:52,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:53,490 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:54,495 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:55,399 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:55,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:56,498 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:57,491 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:57,842 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:50:57,842 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:50:57,842 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:50:58,012 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:50:58,012 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:58,013 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:50:58,263 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/b1z9epwb-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:50:58,385 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:50:58,385 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:50:58,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:59,494 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:00,013 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:00,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:00,536 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:01,495 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:02,014 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:02,502 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:03,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:04,014 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:04,496 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:05,496 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:05,541 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:06,015 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:06,496 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:07,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:08,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:09,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:10,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:11,396 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:11,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:12,498 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:12,998 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:51:13,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:51:13,374 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:51:13,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:14,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:15,498 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:16,498 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:16,517 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:17,498 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:18,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:19,498 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:20,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:21,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:22,397 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:22,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:23,500 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:24,500 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:25,500 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:26,500 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:27,397 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:27,500 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:28,374 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:51:28,374 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:51:28,500 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:29,500 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:30,024 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:30,514 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:31,430 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:51:31,430 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_f02cea2dbceefebb3b97.mp4 with policy now +2025-01-15 21:51:31,446 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:51:31,446 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:51:31,446 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:51:31,447 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:51:31,504 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:31,676 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/hgjwvamf-media/videos/eval/7_eval_7_f02cea2dbceefebb3b97.mp4 +2025-01-15 21:51:32,025 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:32,025 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:51:32,025 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/7_eval_7_f02cea2dbceefebb3b97.mp4 +2025-01-15 21:51:32,025 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:51:32,315 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/dxfpwmx0-media/videos/eval/7_eval_7_f02cea2dbceefebb3b97.mp4 +2025-01-15 21:51:32,492 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:32,520 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:33,520 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:34,026 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:34,525 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:35,522 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:36,027 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:36,526 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:37,526 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:37,982 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:38,541 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:39,529 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:40,029 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:40,541 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:41,543 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:41,897 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:51:41,897 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:51:41,897 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:51:42,030 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:51:42,030 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:51:42,551 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:42,999 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:51:42,999 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:43,382 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:51:43,383 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:51:43,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:44,030 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:44,543 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:45,543 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:46,549 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:47,550 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:48,032 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:48,387 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:48,548 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:49,544 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:50,033 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:50,544 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:51,544 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:52,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:53,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:53,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:54,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:55,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:56,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:57,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:58,378 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:51:58,378 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:51:58,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:58,549 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:59,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:00,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:01,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:02,037 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:02,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:03,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:04,410 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:04,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:05,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:06,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:07,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:08,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:09,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:10,041 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:10,410 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:10,547 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:11,547 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:12,547 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:12,999 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:52:13,378 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:52:13,378 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:52:13,547 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:14,042 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:14,547 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:15,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:15,547 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:16,043 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:16,555 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:16,815 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:52:16,815 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_fb8f14c1c353cc80983a.mp4 with policy now +2025-01-15 21:52:16,829 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:52:16,829 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:52:16,829 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:52:16,830 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:52:17,044 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:52:17,044 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/8_eval_8_fb8f14c1c353cc80983a.mp4 +2025-01-15 21:52:17,044 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:52:17,075 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/xsm8xo6m-media/videos/eval/8_eval_8_fb8f14c1c353cc80983a.mp4 +2025-01-15 21:52:17,344 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/ctcy9ilg-media/videos/eval/8_eval_8_fb8f14c1c353cc80983a.mp4 +2025-01-15 21:52:17,561 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:18,044 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:18,575 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:19,044 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:19,561 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:20,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:20,555 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:21,554 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:22,045 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:22,569 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:23,556 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:24,046 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:24,570 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:25,477 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:25,564 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:26,578 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:27,565 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:28,399 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:52:28,399 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:52:28,587 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:29,572 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:30,483 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:30,584 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:31,576 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:31,960 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:52:31,960 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:52:31,961 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:52:32,049 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:52:32,050 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:32,050 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:52:32,302 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/hj7yr4f7-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:52:32,578 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:33,581 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:34,050 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:34,586 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:35,561 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:35,605 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:36,577 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:37,577 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:38,052 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:38,577 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:39,577 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:40,577 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:41,416 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:41,577 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:42,577 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:43,000 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:52:43,379 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:52:43,379 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:52:43,578 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:44,578 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:45,578 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:46,468 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:46,578 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:47,578 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:48,578 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:49,578 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:50,579 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:51,579 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:51,928 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:52,590 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:53,349 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:52:53,349 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_19795df8831a10c10837.mp4 with policy now +2025-01-15 21:52:53,355 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:52:53,356 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:52:53,356 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:52:53,356 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:52:53,583 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/e9k3rryo-media/videos/eval/9_eval_9_19795df8831a10c10837.mp4 +2025-01-15 21:52:53,585 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:54,058 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:54,058 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:52:54,058 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/9_eval_9_19795df8831a10c10837.mp4 +2025-01-15 21:52:54,059 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:52:54,340 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/61hoxzze-media/videos/eval/9_eval_9_19795df8831a10c10837.mp4 +2025-01-15 21:52:54,592 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:55,058 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:55,595 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:56,597 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:57,047 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:57,625 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:58,060 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:58,399 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:52:58,400 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:52:58,622 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:59,600 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:00,617 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:01,618 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:02,058 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:02,061 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:53:02,061 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:02,612 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:03,000 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:53:03,000 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:53:03,001 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:53:03,061 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:53:03,062 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:53:03,629 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:04,062 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:04,634 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:05,619 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:06,611 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:07,141 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:07,651 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:08,063 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:08,623 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:09,630 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:10,643 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:11,623 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:12,065 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:12,623 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:13,000 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:53:13,002 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:13,379 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:53:13,380 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:53:13,623 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:14,623 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:15,623 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:16,623 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:17,623 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:18,017 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:18,067 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:18,624 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:19,624 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:20,624 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:21,624 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:22,624 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:23,422 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:23,624 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:24,624 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:25,624 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:26,624 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:27,635 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:28,071 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:28,381 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:53:28,381 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_6e43fe2801f3a905e582.mp4 with policy now +2025-01-15 21:53:28,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:53:28,391 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:53:28,391 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:53:28,392 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:53:28,397 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:53:28,397 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:53:28,526 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:28,629 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/jp7098hy-media/videos/eval/10_eval_10_6e43fe2801f3a905e582.mp4 +2025-01-15 21:53:28,630 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:29,072 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:53:29,072 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/10_eval_10_6e43fe2801f3a905e582.mp4 +2025-01-15 21:53:29,072 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:53:29,342 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/utk77r6u-media/videos/eval/10_eval_10_6e43fe2801f3a905e582.mp4 +2025-01-15 21:53:29,646 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:30,072 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:30,634 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:31,073 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:31,666 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:32,642 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:33,647 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:34,074 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:34,432 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:34,649 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:35,656 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:36,075 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:36,657 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:37,044 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:53:37,044 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:53:37,045 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:53:37,075 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:53:37,075 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:53:37,325 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/9thvpq01-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:53:37,668 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:38,075 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:38,659 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:39,458 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:39,662 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:40,660 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:41,672 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:42,077 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:42,672 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:43,001 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:53:43,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:53:43,391 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:53:43,667 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:44,515 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:44,683 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:45,681 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:46,664 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:47,664 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:48,080 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:48,664 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:49,664 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:50,455 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:50,664 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:51,664 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:52,665 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:53,665 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:54,665 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:55,665 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:56,455 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:56,665 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:57,665 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:58,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:53:58,383 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:53:58,665 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:59,666 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:00,666 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:01,500 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:01,666 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:02,709 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:03,692 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:04,043 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:54:04,043 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_4263c7dc48b07a1c552c.mp4 with policy now +2025-01-15 21:54:04,061 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:54:04,062 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:54:04,062 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:54:04,062 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:54:04,086 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:04,086 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:54:04,086 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/11_eval_11_4263c7dc48b07a1c552c.mp4 +2025-01-15 21:54:04,086 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:54:04,323 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/0alu68rb-media/videos/eval/11_eval_11_4263c7dc48b07a1c552c.mp4 +2025-01-15 21:54:04,586 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/wr03doaa-media/videos/eval/11_eval_11_4263c7dc48b07a1c552c.mp4 +2025-01-15 21:54:04,683 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:05,678 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:06,087 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:06,696 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:07,087 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:07,103 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:07,697 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:08,680 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:09,697 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:10,088 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:10,701 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:11,693 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:12,089 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:12,697 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:12,853 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:13,001 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:54:13,399 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:54:13,399 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:54:13,707 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:14,697 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:15,694 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:16,693 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:17,727 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:18,480 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:18,709 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:19,091 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:54:19,096 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:54:19,096 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:54:19,100 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:54:19,730 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:20,092 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:54:20,092 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:20,092 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:54:20,727 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:21,718 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:22,093 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:22,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:23,513 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:23,710 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:24,094 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:24,710 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:25,710 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:26,711 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:27,711 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:28,384 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:54:28,384 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:54:28,712 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:29,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:29,712 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:30,712 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:31,712 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:32,712 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:33,712 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:34,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:35,468 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:35,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:36,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:37,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:38,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:39,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:40,468 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:40,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:41,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:42,714 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:43,002 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:54:43,384 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:54:43,385 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:54:43,714 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:44,714 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:45,489 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:45,714 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:46,714 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:47,714 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:48,714 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:49,714 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:50,729 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:51,053 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:51,725 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:51,773 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:54:51,773 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:54:51,773 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_241be90b3f1c54884e79.mp4 with policy now +2025-01-15 21:54:51,774 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:54:51,774 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:54:51,774 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:54:52,030 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/lmglli64-media/videos/eval/12_eval_12_241be90b3f1c54884e79.mp4 +2025-01-15 21:54:52,105 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:54:52,105 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/12_eval_12_241be90b3f1c54884e79.mp4 +2025-01-15 21:54:52,105 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:54:52,401 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/r6bjjpye-media/videos/eval/12_eval_12_241be90b3f1c54884e79.mp4 +2025-01-15 21:54:52,735 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:53,105 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:53,734 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:54,750 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:55,106 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:55,763 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:56,505 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:56,757 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:57,107 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:57,755 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:58,401 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:54:58,401 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:54:58,751 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:59,763 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:00,761 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:01,108 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:01,522 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:01,776 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:02,109 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:55:02,149 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:55:02,150 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:55:02,150 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:55:02,787 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:03,110 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:03,110 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:55:03,788 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:04,766 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:05,111 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:05,789 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:06,765 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:06,804 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:07,803 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:08,791 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:09,112 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:09,800 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:10,793 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:11,807 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:12,179 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:12,791 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:13,002 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:55:13,388 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:55:13,388 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:55:13,791 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:14,791 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:15,115 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:15,792 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:16,791 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:17,532 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:17,792 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:18,792 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:19,792 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:20,792 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:21,792 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:22,792 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:23,029 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:23,792 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:24,793 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:25,119 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:25,793 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:26,793 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:27,793 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:28,037 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:28,387 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:55:28,387 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:55:28,793 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:29,121 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:29,793 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:30,793 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:31,122 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:31,794 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:32,794 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:33,479 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:33,794 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:34,794 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:35,794 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:36,795 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:37,795 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:38,479 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:38,795 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:39,799 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:40,823 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:40,926 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:55:40,926 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_cb3449f02cfc7cf55b8d.mp4 with policy now +2025-01-15 21:55:40,935 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:55:40,935 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:55:40,936 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:55:40,936 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:55:41,126 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:41,126 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:55:41,126 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/13_eval_13_cb3449f02cfc7cf55b8d.mp4 +2025-01-15 21:55:41,126 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:55:41,174 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/lgjmpjp7-media/videos/eval/13_eval_13_cb3449f02cfc7cf55b8d.mp4 +2025-01-15 21:55:41,428 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/tx0smn3m-media/videos/eval/13_eval_13_cb3449f02cfc7cf55b8d.mp4 +2025-01-15 21:55:41,827 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:42,807 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:43,003 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:55:43,126 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:43,413 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:55:43,413 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:55:43,512 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:43,829 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:44,831 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:45,820 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:46,822 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:47,128 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:47,824 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:48,522 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:48,846 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:49,886 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:50,853 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:51,130 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:51,882 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:52,850 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:53,885 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:54,507 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:54,860 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:55,889 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:56,868 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:57,877 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:58,435 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:55:58,435 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:55:58,881 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:59,133 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:55:59,219 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:55:59,219 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:55:59,219 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:55:59,375 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/t0pun5g6-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:55:59,880 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:00,133 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:56:00,133 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:56:00,274 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:00,888 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:01,134 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:01,871 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:02,906 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:03,134 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:03,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:04,873 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:05,136 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:05,753 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:05,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:06,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:07,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:08,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:09,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:10,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:11,488 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:11,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:12,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:13,005 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:56:13,139 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:13,401 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:56:13,401 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:56:13,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:14,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:15,875 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:16,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:16,875 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:17,875 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:18,875 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:19,875 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:20,875 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:21,142 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:21,876 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:21,933 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:22,876 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:23,143 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:23,876 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:24,876 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:25,876 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:26,876 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:27,145 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:27,876 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:27,926 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:28,401 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:56:28,401 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:56:28,876 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:29,876 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:30,898 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:31,888 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:31,978 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:56:31,978 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_074bdfc80a4a9d765f98.mp4 with policy now +2025-01-15 21:56:31,991 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:56:31,991 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:56:31,991 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:56:31,992 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:56:32,147 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:56:32,148 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/14_eval_14_074bdfc80a4a9d765f98.mp4 +2025-01-15 21:56:32,148 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:56:32,225 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/tyvt3rah-media/videos/eval/14_eval_14_074bdfc80a4a9d765f98.mp4 +2025-01-15 21:56:32,492 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/hvflza3a-media/videos/eval/14_eval_14_074bdfc80a4a9d765f98.mp4 +2025-01-15 21:56:32,891 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:33,088 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:33,148 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:33,890 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:34,912 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:35,149 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:35,908 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:36,914 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:37,939 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:38,515 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:38,925 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:39,151 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:39,936 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:40,917 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:41,152 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:41,940 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:42,966 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:43,003 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:56:43,404 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:56:43,404 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:56:43,960 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:44,508 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:44,954 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:45,942 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:46,938 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:47,950 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:48,943 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:49,517 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:49,935 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:50,940 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:51,978 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:52,951 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:53,960 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:54,537 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:54,962 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:55,969 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:56,988 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:57,973 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:58,427 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:56:58,428 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:56:58,979 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:00,018 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:00,528 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:01,023 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:02,025 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:02,992 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:03,987 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:04,997 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:05,537 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:05,988 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:07,004 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:08,017 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:09,027 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:10,024 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:11,027 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:11,524 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:12,040 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:13,004 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:57:13,026 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:13,419 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:57:13,420 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:57:14,035 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:15,023 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:16,020 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:17,022 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:17,523 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:18,030 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:19,024 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:20,074 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:21,067 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:22,043 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:22,526 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:23,053 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:24,055 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:25,059 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:26,064 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:27,050 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:27,554 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:28,069 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:28,432 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:57:28,432 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:57:29,077 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:30,066 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:31,064 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:32,083 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:32,556 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:33,096 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:34,070 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:35,094 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:36,072 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:37,109 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:38,089 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:38,520 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:39,105 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:40,102 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:41,095 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:42,099 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:43,004 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:57:43,126 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:43,426 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:57:43,426 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:57:43,539 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:44,111 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:45,127 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:46,118 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:47,118 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:48,122 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:49,113 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:49,526 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:50,125 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:51,123 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:52,120 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:53,150 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:54,153 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:54,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:55,156 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:56,145 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:57,134 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:58,160 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:58,425 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:57:58,425 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:57:59,155 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:59,547 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:00,146 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:01,182 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:02,151 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:03,167 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:04,180 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:04,555 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:05,170 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:06,172 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:07,179 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:08,174 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:09,194 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:10,232 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:10,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:11,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:12,205 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:13,005 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:58:13,210 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:13,453 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:58:13,453 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:58:14,246 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:15,245 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:15,559 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:16,265 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:17,226 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:18,234 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:19,234 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:20,234 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:20,601 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:21,239 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:22,237 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:23,242 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:24,244 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:25,273 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:26,262 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:26,571 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:27,270 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:28,251 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:28,431 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:58:28,431 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:58:29,264 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:30,285 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:31,278 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:31,580 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:32,280 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:33,283 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:34,308 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:35,318 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:36,352 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:37,309 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:37,579 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:38,309 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:39,343 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:40,313 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:41,311 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:42,321 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:42,640 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:43,005 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:58:43,329 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:43,464 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:58:43,464 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:58:44,347 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:45,339 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:46,342 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:47,342 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:48,339 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:48,596 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:49,378 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:50,358 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:51,359 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:52,345 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:53,358 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:53,604 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:54,354 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:55,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:56,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:57,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:58,372 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:58,448 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:58:58,448 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:58:58,645 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:59,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:00,378 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:01,372 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:02,401 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:03,401 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:04,402 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:04,608 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:05,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:06,406 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:07,426 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:08,431 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:09,417 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:10,429 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:10,608 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:11,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:12,431 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:13,006 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:59:13,451 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:13,463 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:59:13,463 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:59:14,465 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:15,454 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:15,619 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:16,428 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:17,456 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:18,457 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:19,455 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:20,461 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:20,621 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:21,448 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:22,487 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:23,495 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:24,496 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:25,452 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:26,458 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:26,614 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:27,481 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:28,453 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:59:28,453 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:59:28,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:29,475 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:30,461 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:31,478 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:31,645 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:32,474 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:33,469 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:34,490 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:35,530 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:36,512 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:37,505 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:37,623 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:38,526 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:39,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:40,531 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:41,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:42,563 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:42,642 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:43,006 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:59:43,461 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:59:43,461 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:59:43,566 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:44,557 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:45,558 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:46,561 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:47,565 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:48,595 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:48,626 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:49,569 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:50,575 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:51,589 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:52,588 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:53,583 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:54,590 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:54,626 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:55,586 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:56,602 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:57,591 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:58,461 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:59:58,461 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:59:58,589 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:59,605 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:59,643 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:00,616 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:01,609 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:02,237 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:00:02,498 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:00:02,498 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:00:02,498 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:00:02,617 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:03,238 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:03,238 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:00:03,614 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:04,633 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:04,644 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:05,239 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:05,611 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:06,611 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:07,617 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:08,628 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:09,241 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:09,647 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:09,649 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:10,619 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:11,619 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:12,638 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:13,006 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:00:13,473 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:00:13,473 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:00:13,630 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:14,621 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:15,243 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:15,278 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:15,621 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:16,621 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:17,244 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:17,621 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:18,621 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:19,621 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:20,622 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:20,630 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:21,622 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:22,622 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:23,622 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:24,622 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:25,622 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:25,630 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:26,622 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:27,622 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:28,466 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:00:28,466 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:00:28,667 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:29,249 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:29,659 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:30,007 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 22:00:30,007 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_76fa4e81e9e73fef9cf0.mp4 with policy now +2025-01-15 22:00:30,032 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 22:00:30,032 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:00:30,033 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:00:30,033 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:00:30,249 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:00:30,250 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/15_eval_15_76fa4e81e9e73fef9cf0.mp4 +2025-01-15 22:00:30,250 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 22:00:30,263 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/858lh38v-media/videos/eval/15_eval_15_76fa4e81e9e73fef9cf0.mp4 +2025-01-15 22:00:30,537 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/pt3wspel-media/videos/eval/15_eval_15_76fa4e81e9e73fef9cf0.mp4 +2025-01-15 22:00:30,675 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:31,064 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:31,250 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:31,670 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:32,669 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:33,251 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:33,737 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:34,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:35,252 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:35,716 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:36,660 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:36,711 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:37,720 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:38,719 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:39,253 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:39,712 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:40,722 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:41,673 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:41,741 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:42,720 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:43,007 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:00:43,473 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:00:43,474 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:00:43,741 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:44,737 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:45,256 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:00:45,532 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/ia05v5to-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:00:45,555 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:00:45,555 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:00:45,555 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:00:45,744 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:46,256 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:00:46,732 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:46,953 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:47,257 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:47,734 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:48,744 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:49,747 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:50,777 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:51,258 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:51,768 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:52,187 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:52,748 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:53,767 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:54,769 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:55,781 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:56,783 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:57,670 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:57,802 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:58,454 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:00:58,455 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:00:58,759 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:59,262 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:59,759 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:00,759 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:01,263 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:01:01,761 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:02,760 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:03,648 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:03,760 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:04,761 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:05,761 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:06,761 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:07,761 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:08,648 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:08,761 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:09,762 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:10,761 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:11,762 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:12,762 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:13,007 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:01:13,454 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:01:13,454 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:01:13,648 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:13,762 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:14,762 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:15,763 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:16,763 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:17,763 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:18,649 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:18,763 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:19,763 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:20,764 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:21,764 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:22,764 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:23,649 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:23,764 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:24,764 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:25,813 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:26,274 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/16_eval_16_5019757f76965e10782b.mp4 +2025-01-15 22:01:26,274 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 22:01:26,287 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 22:01:26,288 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_5019757f76965e10782b.mp4 with policy now +2025-01-15 22:01:26,290 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 22:01:26,291 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:01:26,291 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:01:26,291 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:01:26,545 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/7disiecn-media/videos/eval/16_eval_16_5019757f76965e10782b.mp4 +2025-01-15 22:01:26,798 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:26,836 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/xc8psx8a-media/videos/eval/16_eval_16_5019757f76965e10782b.mp4 +2025-01-15 22:01:27,274 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:01:27,274 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:01:27,813 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:28,487 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:01:28,488 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:01:28,654 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:28,808 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:29,275 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:01:29,826 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:30,815 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:31,805 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:32,810 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:33,277 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:01:33,855 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:34,653 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:34,829 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:35,278 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:01:35,810 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:36,840 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:37,865 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:38,845 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:39,685 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:39,845 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:40,858 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:41,836 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:42,848 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:43,008 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:01:43,492 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:01:43,492 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:01:43,843 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:44,860 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:45,684 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:45,867 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:46,886 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:47,900 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:48,864 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:49,926 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:50,700 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:50,886 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:51,890 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:52,897 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:53,897 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:54,905 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:55,955 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:56,694 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:56,909 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:57,915 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:58,478 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:01:58,479 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:01:58,907 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:59,928 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:00,944 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:01,709 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:01,982 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:02,985 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:03,949 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:04,981 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:05,959 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:06,716 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:06,974 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:07,970 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:08,969 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:09,979 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:10,981 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:11,988 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:12,694 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:12,994 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:13,008 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:02:13,480 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:02:13,480 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:02:14,007 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:15,012 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:15,295 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:02:15,674 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:02:15,674 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:02:15,674 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:02:16,008 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:16,295 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:02:17,044 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:17,296 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:02:18,006 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:18,129 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:19,023 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:20,005 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:21,005 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:21,297 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:02:22,005 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:23,005 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:23,539 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:24,005 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:25,006 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:26,006 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:27,006 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:28,006 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:28,469 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:02:28,469 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:02:28,615 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:29,006 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:30,006 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:31,007 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:32,008 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:33,008 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:33,692 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:34,031 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:35,071 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:35,305 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:02:35,410 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 22:02:35,410 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_2f96b121fd6f99c60674.mp4 with policy now +2025-01-15 22:02:35,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 22:02:35,423 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:02:35,423 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:02:35,424 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:02:35,646 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/4yu7df0h-media/videos/eval/17_eval_17_2f96b121fd6f99c60674.mp4 +2025-01-15 22:02:36,063 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:36,306 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:02:36,306 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/17_eval_17_2f96b121fd6f99c60674.mp4 +2025-01-15 22:02:36,306 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 22:02:36,599 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/t1zmllnw-media/videos/eval/17_eval_17_2f96b121fd6f99c60674.mp4 +2025-01-15 22:02:37,044 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:37,306 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:02:38,080 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:38,731 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:39,059 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:40,076 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:41,061 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:41,308 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:02:42,069 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:43,009 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:02:43,089 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:43,507 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:02:43,508 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:02:44,099 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:44,730 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:45,113 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:45,310 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:02:46,115 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:47,119 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:48,115 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:49,135 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:49,738 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:50,127 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:51,153 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:52,146 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:53,151 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:54,157 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:54,741 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:55,171 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:56,148 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:57,153 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:58,197 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:58,315 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:02:58,482 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:02:58,483 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:02:58,725 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:02:58,725 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:02:58,725 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:02:59,185 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:59,316 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:02:59,316 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:02:59,316 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:03:00,173 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:00,401 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:01,164 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:01,317 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:02,188 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:03,191 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:04,190 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:05,173 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:05,318 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:05,734 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:06,203 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:07,205 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:08,201 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:09,207 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:10,194 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:11,134 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:11,194 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:11,321 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:12,194 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:13,009 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:03:13,194 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:13,478 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:03:13,478 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:03:14,194 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:15,195 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:16,195 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:16,789 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:17,195 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:18,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:19,195 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:20,195 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:21,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:22,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:22,723 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:23,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:24,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:25,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:26,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:27,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:27,723 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:28,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:28,478 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:03:28,479 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:03:29,197 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:30,197 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:31,197 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:32,197 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:32,723 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:33,197 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:34,197 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:35,197 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:36,197 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:37,212 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:37,725 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:38,199 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:38,518 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 22:03:38,518 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_c8c84019c8dc8ba0e4ea.mp4 with policy now +2025-01-15 22:03:38,526 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 22:03:38,526 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:03:38,526 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:03:38,527 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:03:38,777 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/n4px6l3n-media/videos/eval/18_eval_18_c8c84019c8dc8ba0e4ea.mp4 +2025-01-15 22:03:39,210 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:39,333 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:39,333 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:03:39,334 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/18_eval_18_c8c84019c8dc8ba0e4ea.mp4 +2025-01-15 22:03:39,334 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 22:03:39,649 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/szqdmjnl-media/videos/eval/18_eval_18_c8c84019c8dc8ba0e4ea.mp4 +2025-01-15 22:03:40,215 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:41,221 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:41,334 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:42,244 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:43,009 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:03:43,011 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:43,228 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:43,532 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:03:43,532 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:03:44,260 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:45,241 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:45,336 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:46,249 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:47,261 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:47,337 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:48,263 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:48,729 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:49,249 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:50,255 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:51,341 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:52,275 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:53,273 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:53,339 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:03:53,787 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:53,791 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:03:53,791 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:03:53,792 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:03:54,283 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:54,340 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:03:54,340 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:03:55,297 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:55,340 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:56,326 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:57,278 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:57,341 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:58,279 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:58,517 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:03:58,518 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:03:59,294 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:59,342 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:59,768 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:00,292 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:01,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:02,344 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:03,329 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:04,326 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:04,779 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:05,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:06,313 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:07,348 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:08,340 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:09,388 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:09,800 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:10,316 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:11,315 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:11,347 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:04:12,316 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:13,010 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:04:13,316 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:13,492 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:04:13,492 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:04:14,317 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:15,317 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:15,760 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:16,317 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:17,317 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:18,317 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:19,317 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:20,317 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:21,317 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:21,760 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:22,318 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:23,318 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:24,318 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:25,318 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:26,318 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:26,760 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:27,318 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:28,318 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:28,492 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:04:28,492 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:04:29,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:30,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:31,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:31,760 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:32,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:33,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:34,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:35,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:36,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:37,352 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:37,439 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:38,339 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 22:04:38,339 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_3f737e19fed74dd298b2.mp4 with policy now +2025-01-15 22:04:38,359 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/19_eval_19_3f737e19fed74dd298b2.mp4 +2025-01-15 22:04:38,359 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 22:04:38,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 22:04:38,377 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:04:38,377 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:38,377 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:38,379 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:38,606 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/tje53cgu-media/videos/eval/19_eval_19_3f737e19fed74dd298b2.mp4 +2025-01-15 22:04:38,908 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/s9jrxm23-media/videos/eval/19_eval_19_3f737e19fed74dd298b2.mp4 +2025-01-15 22:04:39,357 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:39,360 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:04:39,360 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:04:40,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:41,361 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:04:41,368 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:42,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:43,010 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:04:43,011 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:43,361 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:04:43,403 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:43,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:04:43,499 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:04:44,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:45,389 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:46,390 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:47,363 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:04:47,411 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:48,379 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:48,625 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:49,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:50,402 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:51,413 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:52,406 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:53,439 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:53,794 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:54,366 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:04:54,437 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:54,642 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/gms0en9u-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:04:54,864 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:04:54,864 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:54,864 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:55,367 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:04:55,367 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:04:55,367 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:04:55,446 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:55,970 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-15 22:04:55,971 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: log_artifact +2025-01-15 22:04:56,367 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table/eval/success_rate_ci_20_0f278608fd5017506fe4.table.json +2025-01-15 22:04:56,367 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media +2025-01-15 22:04:56,368 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table +2025-01-15 22:04:56,368 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table/eval +2025-01-15 22:04:56,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:56,449 INFO wandb-upload_3:2470873 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpnbdac6qz +2025-01-15 22:04:56,476 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 22:04:56,568 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/5fe6wu6w-media/table/eval/success_rate_ci_20_0f278608fd5017506fe4.table.json +2025-01-15 22:04:56,836 INFO SenderThread:2470873 [sender.py:send_request_log_artifact():1525] logged artifact run-roatk2bk-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0NDEwODI1NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzOTE1MDQ5NA==', 'latestArtifact': None}} +2025-01-15 22:04:56,836 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 22:04:56,836 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_0f278608fd5017506fe4.table.json with policy now +2025-01-15 22:04:56,836 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:04:56,836 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,837 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,837 DEBUG SenderThread:2470873 [sender.py:send():379] send: telemetry +2025-01-15 22:04:56,837 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,838 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,838 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,838 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,839 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,840 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,841 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,841 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,841 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,842 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,842 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,842 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,842 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,843 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,843 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,843 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,844 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,844 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,844 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,845 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,845 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,845 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,875 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-15 22:04:56,875 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: log_artifact +2025-01-15 22:04:56,887 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 22:04:57,140 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/ve85kqna-media/table/eval/success_rate_ci_20_0f278608fd5017506fe4.table.json +2025-01-15 22:04:57,368 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:04:57,368 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:04:57,368 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table/eval/total_success_rate_ci_21_ee38b3327b46c4981cc4.table.json +2025-01-15 22:04:57,368 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table/eval +2025-01-15 22:04:57,389 INFO wandb-upload_3:2470873 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpuhxzzwc7 +2025-01-15 22:04:57,406 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:57,563 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/44kyffge-media/table/eval/total_success_rate_ci_21_ee38b3327b46c4981cc4.table.json +2025-01-15 22:04:57,762 INFO SenderThread:2470873 [sender.py:send_request_log_artifact():1525] logged artifact run-roatk2bk-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0NDEwODI2Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzOTE1MDQ5Ng==', 'latestArtifact': None}} +2025-01-15 22:04:57,762 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 22:04:57,762 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_ee38b3327b46c4981cc4.table.json with policy now +2025-01-15 22:04:57,763 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:04:57,763 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:57,763 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:58,031 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/ydokvtt1-media/table/eval/total_success_rate_ci_21_ee38b3327b46c4981cc4.table.json +2025-01-15 22:04:58,368 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:04:58,406 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:58,496 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:04:58,496 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:04:58,798 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:59,369 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:04:59,406 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:00,406 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:01,406 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:02,406 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:03,406 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:03,802 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:05:04,372 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/config.yaml +2025-01-15 22:05:04,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:05,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:06,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:07,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:08,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:09,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:09,799 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:05:10,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:11,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:12,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:13,011 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:05:13,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:13,496 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:05:13,496 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:05:14,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:14,841 DEBUG SenderThread:2470873 [sender.py:send():379] send: exit +2025-01-15 22:05:14,841 INFO SenderThread:2470873 [sender.py:send_exit():586] handling exit code: 0 +2025-01-15 22:05:14,842 INFO SenderThread:2470873 [sender.py:send_exit():588] handling runtime: 1201 +2025-01-15 22:05:14,842 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:05:14,842 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:05:14,842 INFO SenderThread:2470873 [sender.py:send_exit():594] send defer +2025-01-15 22:05:14,842 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:14,842 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-15 22:05:14,843 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:14,843 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-15 22:05:14,843 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 1 +2025-01-15 22:05:14,843 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:14,843 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-15 22:05:14,843 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:14,843 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-15 22:05:14,843 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 2 +2025-01-15 22:05:14,843 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:14,843 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-15 22:05:14,843 INFO HandlerThread:2470873 [system_monitor.py:finish():203] Stopping system monitor +2025-01-15 22:05:14,843 DEBUG SystemMonitor:2470873 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-15 22:05:14,844 INFO HandlerThread:2470873 [interfaces.py:finish():200] Joined cpu monitor +2025-01-15 22:05:14,844 DEBUG SystemMonitor:2470873 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-15 22:05:14,844 INFO HandlerThread:2470873 [interfaces.py:finish():200] Joined disk monitor +2025-01-15 22:05:14,915 INFO HandlerThread:2470873 [interfaces.py:finish():200] Joined gpu monitor +2025-01-15 22:05:14,915 INFO HandlerThread:2470873 [interfaces.py:finish():200] Joined memory monitor +2025-01-15 22:05:14,915 INFO HandlerThread:2470873 [interfaces.py:finish():200] Joined network monitor +2025-01-15 22:05:14,915 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:14,915 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-15 22:05:14,916 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 3 +2025-01-15 22:05:14,916 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:05:14,916 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:14,916 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-15 22:05:14,916 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:14,916 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-15 22:05:14,916 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 4 +2025-01-15 22:05:14,916 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:14,916 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-15 22:05:15,378 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:05:17,379 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:05:21,872 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 22:05:21,873 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:05:21,873 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: keepalive +2025-01-15 22:05:21,874 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:21,874 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-15 22:05:21,874 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 5 +2025-01-15 22:05:21,874 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 22:05:21,874 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:05:21,874 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:05:21,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:21,875 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-15 22:05:21,875 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:05:21,875 DEBUG SenderThread:2470873 [sender.py:send():379] send: summary +2025-01-15 22:05:21,879 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:05:21,879 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:21,879 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-15 22:05:21,879 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 6 +2025-01-15 22:05:21,879 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:21,880 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-15 22:05:21,880 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:21,880 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-15 22:05:21,880 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 7 +2025-01-15 22:05:21,880 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:05:21,880 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:21,880 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-15 22:05:21,880 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:21,880 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-15 22:05:22,381 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:05:22,843 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 22:05:23,031 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 8 +2025-01-15 22:05:23,031 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 22:05:23,031 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:23,031 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-15 22:05:23,031 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:23,032 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-15 22:05:23,032 INFO SenderThread:2470873 [job_builder.py:build():440] Attempting to build job artifact +2025-01-15 22:05:23,035 INFO SenderThread:2470873 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-15 22:05:23,036 INFO SenderThread:2470873 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-15 22:05:23,049 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 9 +2025-01-15 22:05:23,049 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:23,049 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-15 22:05:23,050 DEBUG SenderThread:2470873 [sender.py:send():379] send: artifact +2025-01-15 22:05:23,221 INFO SenderThread:2470873 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ0NDA3MjMyOQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ0NDA3MjMyOQ==', 'versionIndex': 440}}} +2025-01-15 22:05:23,221 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:23,221 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-15 22:05:23,221 INFO SenderThread:2470873 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-15 22:05:23,382 INFO SenderThread:2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:05:23,383 INFO SenderThread:2470873 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files +2025-01-15 22:05:23,383 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/config.yaml config.yaml +2025-01-15 22:05:23,383 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json wandb-summary.json +2025-01-15 22:05:23,383 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/requirements.txt requirements.txt +2025-01-15 22:05:23,386 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-metadata.json wandb-metadata.json +2025-01-15 22:05:23,386 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:05:23,386 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-15 22:05:23,386 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log output.log +2025-01-15 22:05:23,388 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/diff.patch diff.patch +2025-01-15 22:05:23,388 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-15 22:05:23,388 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/3_eval_3_89821fb09b372630838b.mp4 media/videos/eval/3_eval_3_89821fb09b372630838b.mp4 +2025-01-15 22:05:23,388 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/4_eval_4_9fb52356d0a213033cfe.mp4 media/videos/eval/4_eval_4_9fb52356d0a213033cfe.mp4 +2025-01-15 22:05:23,388 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/10_eval_10_6e43fe2801f3a905e582.mp4 media/videos/eval/10_eval_10_6e43fe2801f3a905e582.mp4 +2025-01-15 22:05:23,388 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/5_eval_5_a69fcdfd49e2066116b6.mp4 media/videos/eval/5_eval_5_a69fcdfd49e2066116b6.mp4 +2025-01-15 22:05:23,388 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/0_eval_0_2eb6cf19687a4fabfe3c.mp4 media/videos/eval/0_eval_0_2eb6cf19687a4fabfe3c.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/18_eval_18_c8c84019c8dc8ba0e4ea.mp4 media/videos/eval/18_eval_18_c8c84019c8dc8ba0e4ea.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/15_eval_15_76fa4e81e9e73fef9cf0.mp4 media/videos/eval/15_eval_15_76fa4e81e9e73fef9cf0.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/19_eval_19_3f737e19fed74dd298b2.mp4 media/videos/eval/19_eval_19_3f737e19fed74dd298b2.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/17_eval_17_2f96b121fd6f99c60674.mp4 media/videos/eval/17_eval_17_2f96b121fd6f99c60674.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/12_eval_12_241be90b3f1c54884e79.mp4 media/videos/eval/12_eval_12_241be90b3f1c54884e79.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/13_eval_13_cb3449f02cfc7cf55b8d.mp4 media/videos/eval/13_eval_13_cb3449f02cfc7cf55b8d.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/1_eval_1_b5d7870b7b04a6ef485f.mp4 media/videos/eval/1_eval_1_b5d7870b7b04a6ef485f.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/2_eval_2_f98c31cd2e05324cffd8.mp4 media/videos/eval/2_eval_2_f98c31cd2e05324cffd8.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/6_eval_6_ae418a6c99a51b4201d6.mp4 media/videos/eval/6_eval_6_ae418a6c99a51b4201d6.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/14_eval_14_074bdfc80a4a9d765f98.mp4 media/videos/eval/14_eval_14_074bdfc80a4a9d765f98.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/16_eval_16_5019757f76965e10782b.mp4 media/videos/eval/16_eval_16_5019757f76965e10782b.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/7_eval_7_f02cea2dbceefebb3b97.mp4 media/videos/eval/7_eval_7_f02cea2dbceefebb3b97.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/9_eval_9_19795df8831a10c10837.mp4 media/videos/eval/9_eval_9_19795df8831a10c10837.mp4 +2025-01-15 22:05:23,390 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/8_eval_8_fb8f14c1c353cc80983a.mp4 media/videos/eval/8_eval_8_fb8f14c1c353cc80983a.mp4 +2025-01-15 22:05:23,390 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/11_eval_11_4263c7dc48b07a1c552c.mp4 media/videos/eval/11_eval_11_4263c7dc48b07a1c552c.mp4 +2025-01-15 22:05:23,390 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table/eval/total_success_rate_ci_21_ee38b3327b46c4981cc4.table.json media/table/eval/total_success_rate_ci_21_ee38b3327b46c4981cc4.table.json +2025-01-15 22:05:23,390 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table/eval/success_rate_ci_20_0f278608fd5017506fe4.table.json media/table/eval/success_rate_ci_20_0f278608fd5017506fe4.table.json +2025-01-15 22:05:23,390 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 10 +2025-01-15 22:05:23,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:23,393 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-15 22:05:23,393 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:23,393 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-15 22:05:23,393 INFO SenderThread:2470873 [file_pusher.py:finish():169] shutting down file pusher +2025-01-15 22:05:23,569 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:05:23,575 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/config.yaml +2025-01-15 22:05:23,597 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/requirements.txt +2025-01-15 22:05:23,660 INFO wandb-upload_4:2470873 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:05:23,707 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/niuudu4q-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:05:23,843 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 22:05:23,843 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 22:05:23,908 INFO Thread-11 (_thread_body):2470873 [sender.py:transition_state():614] send defer: 11 +2025-01-15 22:05:23,908 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:23,908 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-15 22:05:23,908 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:23,908 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-15 22:05:23,908 INFO SenderThread:2470873 [file_pusher.py:join():175] waiting for file pusher +2025-01-15 22:05:23,916 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 12 +2025-01-15 22:05:23,916 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:23,916 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-15 22:05:23,916 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:23,916 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-15 22:05:23,916 INFO SenderThread:2470873 [file_stream.py:finish():601] file stream finish called +2025-01-15 22:05:24,132 INFO SenderThread:2470873 [file_stream.py:finish():605] file stream finish is done +2025-01-15 22:05:24,132 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 13 +2025-01-15 22:05:24,133 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:24,133 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-15 22:05:24,133 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:24,133 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-15 22:05:24,133 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 14 +2025-01-15 22:05:24,133 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:24,133 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-15 22:05:24,133 DEBUG SenderThread:2470873 [sender.py:send():379] send: final +2025-01-15 22:05:24,133 DEBUG SenderThread:2470873 [sender.py:send():379] send: footer +2025-01-15 22:05:24,133 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:24,133 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-15 22:05:24,134 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 22:05:24,134 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: server_info +2025-01-15 22:05:24,134 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 22:05:24,134 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 22:05:24,135 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: server_info +2025-01-15 22:05:24,136 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: get_summary +2025-01-15 22:05:24,137 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-15 22:05:24,138 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:24,187 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 22:05:24,188 INFO MainThread:2470873 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-15 22:05:24,188 INFO MainThread:2470873 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-15 22:05:24,189 INFO MainThread:2470873 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-15 22:05:24,189 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: shutdown +2025-01-15 22:05:24,189 INFO HandlerThread:2470873 [handler.py:finish():882] shutting down handler +2025-01-15 22:05:25,135 INFO WriterThread:2470873 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/run-roatk2bk.wandb +2025-01-15 22:05:25,188 INFO SenderThread:2470873 [sender.py:finish():1615] shutting down sender +2025-01-15 22:05:25,188 INFO SenderThread:2470873 [file_pusher.py:finish():169] shutting down file pusher +2025-01-15 22:05:25,188 INFO SenderThread:2470873 [file_pusher.py:join():175] waiting for file pusher diff --git a/roatk2bk/wandb/debug.log b/roatk2bk/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..90333c4a07365b51ebf0a693ef911a3a0ac455c9 --- /dev/null +++ b/roatk2bk/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-15 21:45:12,618 INFO MainThread:2470707 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-15 21:45:12,618 INFO MainThread:2470707 [wandb_setup.py:_flush():76] Configure stats pid to 2470707 +2025-01-15 21:45:12,618 INFO MainThread:2470707 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/settings +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/logs/debug.log +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/logs/debug-internal.log +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_init.py:init():569] calling init triggers +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3456_0', 'wandb_notes': '3456_0_req_3921_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk'} +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_init.py:init():619] starting backend +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_init.py:init():623] setting up manager +2025-01-15 21:45:12,623 INFO MainThread:2470707 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-15 21:45:12,624 INFO MainThread:2470707 [wandb_init.py:init():631] backend started and connected +2025-01-15 21:45:12,637 INFO MainThread:2470707 [wandb_init.py:init():720] updated telemetry +2025-01-15 21:45:12,642 INFO MainThread:2470707 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-15 21:45:12,911 INFO MainThread:2470707 [wandb_run.py:_on_init():2435] communicating current version +2025-01-15 21:45:12,981 INFO MainThread:2470707 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.3 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-15 21:45:12,981 INFO MainThread:2470707 [wandb_init.py:init():804] starting run threads in backend +2025-01-15 21:45:13,359 INFO MainThread:2470707 [wandb_run.py:_console_start():2413] atexit reg +2025-01-15 21:45:13,359 INFO MainThread:2470707 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-15 21:45:13,359 INFO MainThread:2470707 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-15 21:45:13,359 INFO MainThread:2470707 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-15 21:45:13,363 INFO MainThread:2470707 [wandb_init.py:init():847] run started, returning control to user process +2025-01-15 21:45:13,363 INFO MainThread:2470707 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/tb, True +2025-01-15 21:45:41,677 INFO MainThread:2470707 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-15 22:05:25,189 WARNING MsgRouterThr:2470707 [router.py:message_loop():77] message_loop has been closed diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/code/FISH/eval_robot.py b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/config.yaml b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8116a3f32815d7f45924a53a0ffd497546052324 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/config.yaml @@ -0,0 +1,927 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '3456_0' +wandb_notes: + desc: null + value: 3456_0_req_3921_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: null +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 20 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736995512 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/diff.patch b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..6a47c0ed68a9c65b71fd74d87bee4cd94105b0dd --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/diff.patch @@ -0,0 +1,74 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0b377ae..2659f59 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,7 +133,7 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1002_0' ++wandb_run_id: '3456_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..134df65 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,12 +167,14 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ '3456_0', ++ '3456_1', + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table/eval/success_rate_ci_20_0f278608fd5017506fe4.table.json b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table/eval/success_rate_ci_20_0f278608fd5017506fe4.table.json new file mode 100644 index 0000000000000000000000000000000000000000..be6cf800cfd243e4030d24a3053812e9c2bc5943 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table/eval/success_rate_ci_20_0f278608fd5017506fe4.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.6, 0.3280967474327571, 0.8194341956930253, 0.3644341677698424, 0.7783664017358993, 0.4294946342847692, 0.7635830342242457, 0.4492693543535604, 0.7487341165045135]]} \ No newline at end of file diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table/eval/total_success_rate_ci_21_ee38b3327b46c4981cc4.table.json b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table/eval/total_success_rate_ci_21_ee38b3327b46c4981cc4.table.json new file mode 100644 index 0000000000000000000000000000000000000000..6e255d3b34ba24cd963522ea7b3937a4750cf158 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table/eval/total_success_rate_ci_21_ee38b3327b46c4981cc4.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[27, 40, 0.675, 0.4954639971265328, 0.827632218533793, 0.5256678312965555, 0.7974396570922111, 0.558896869409226, 0.7784236967006781, 0.5626946240538138, 0.7666594795349383]]} \ No newline at end of file diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/0_eval_0_2eb6cf19687a4fabfe3c.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/0_eval_0_2eb6cf19687a4fabfe3c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4e04c623aff9ed8198593d50fcc11f61c91475be --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/0_eval_0_2eb6cf19687a4fabfe3c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2eb6cf19687a4fabfe3c6f8e612266c1da2cc0076d60b65057514a926a74144f +size 660308 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/10_eval_10_6e43fe2801f3a905e582.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/10_eval_10_6e43fe2801f3a905e582.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c44c0857412c3888597fe9729489631f824fef7e --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/10_eval_10_6e43fe2801f3a905e582.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e43fe2801f3a905e582de81a9680045e1a4067f03042a5ff19f96803aa6f698 +size 1031684 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/11_eval_11_4263c7dc48b07a1c552c.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/11_eval_11_4263c7dc48b07a1c552c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..82ee335b004ac11441bdf08b4e71f471bf38ec75 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/11_eval_11_4263c7dc48b07a1c552c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4263c7dc48b07a1c552cc54d84f88093f2e788ee860e0a2d14e44c2cc1fdaeef +size 1084332 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/12_eval_12_241be90b3f1c54884e79.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/12_eval_12_241be90b3f1c54884e79.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6e4c28c10969c11aef3116208dd2ee0b3770d95f --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/12_eval_12_241be90b3f1c54884e79.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:241be90b3f1c54884e79ee6f66e49e2b9cd981f6a46f20431ce6ffa92227c5b2 +size 1167206 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/13_eval_13_cb3449f02cfc7cf55b8d.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/13_eval_13_cb3449f02cfc7cf55b8d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7505f4a66141c528032949a427cc5bfbe03ece02 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/13_eval_13_cb3449f02cfc7cf55b8d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb3449f02cfc7cf55b8d5c8b2858c5450855681d80b52de97d006761f8ae751b +size 1419465 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/14_eval_14_074bdfc80a4a9d765f98.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/14_eval_14_074bdfc80a4a9d765f98.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f9062562c7d4ae9e368c27df289d0faf1528f9bb --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/14_eval_14_074bdfc80a4a9d765f98.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:074bdfc80a4a9d765f986d8266f086e36f6336cbfc94d03d20f82b5898467960 +size 1166864 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/15_eval_15_76fa4e81e9e73fef9cf0.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/15_eval_15_76fa4e81e9e73fef9cf0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0be7b49ee55c4e0d9cab9ef43d8fde613a9c5e87 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/15_eval_15_76fa4e81e9e73fef9cf0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76fa4e81e9e73fef9cf0cbf0cae5de5fadd030e8577b710e5fbbc635395fbbad +size 961267 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/16_eval_16_5019757f76965e10782b.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/16_eval_16_5019757f76965e10782b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..35878cc45c89c73ccaf9e56f66bf614e34393ad0 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/16_eval_16_5019757f76965e10782b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5019757f76965e10782b8dbf76cb0ec5dd9466a48a2c73f37eaae45a01df3902 +size 1505807 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/17_eval_17_2f96b121fd6f99c60674.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/17_eval_17_2f96b121fd6f99c60674.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..415050cdcbaa3f4f78d6645b8f7e282c5e7e233d --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/17_eval_17_2f96b121fd6f99c60674.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f96b121fd6f99c60674db7dbf14916a988cd7103feb371d1024b26d7d81e965 +size 680421 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/18_eval_18_c8c84019c8dc8ba0e4ea.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/18_eval_18_c8c84019c8dc8ba0e4ea.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..85b9b2104bf48e99768a588bc493e0090c07dd93 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/18_eval_18_c8c84019c8dc8ba0e4ea.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8c84019c8dc8ba0e4ead689259df8c2ee18d6119c191be15ecdb4cd11fc2f0f +size 1441508 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/19_eval_19_3f737e19fed74dd298b2.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/19_eval_19_3f737e19fed74dd298b2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a8251c6e3499a0a662f47571fb875853fef7ac5b --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/19_eval_19_3f737e19fed74dd298b2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f737e19fed74dd298b263bf46de6d1133a6a7db350c8d12a1fdd73af013c15b +size 1623338 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/1_eval_1_b5d7870b7b04a6ef485f.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/1_eval_1_b5d7870b7b04a6ef485f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b7b4fbb505f3c7a6024f46f7e3eb58f3fc9abc15 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/1_eval_1_b5d7870b7b04a6ef485f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5d7870b7b04a6ef485f99d3b1ba052460f021a02fc0a4bcb269c4a104c53731 +size 1158221 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/2_eval_2_f98c31cd2e05324cffd8.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/2_eval_2_f98c31cd2e05324cffd8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..181e6cebd1809a6788d1392e25ffdddd9d6b2a58 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/2_eval_2_f98c31cd2e05324cffd8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f98c31cd2e05324cffd88658a9b1b179814eff0305d38aba54e59921246de742 +size 810213 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/3_eval_3_89821fb09b372630838b.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/3_eval_3_89821fb09b372630838b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..de47f098ac8dfb66520fa77a28e642d4ee0c52c4 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/3_eval_3_89821fb09b372630838b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89821fb09b372630838b8abd67ec78c34acf566da143b01acc2d08888bb0bc6c +size 1153488 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/4_eval_4_9fb52356d0a213033cfe.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/4_eval_4_9fb52356d0a213033cfe.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..be7d633e95c3b15759f18219f17205e3d270d114 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/4_eval_4_9fb52356d0a213033cfe.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fb52356d0a213033cfe25b61a1934cbe1ee32f73cd0b718905313f2644136c0 +size 897581 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/5_eval_5_a69fcdfd49e2066116b6.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/5_eval_5_a69fcdfd49e2066116b6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f9ea809294c8115d5a59122311bd41632ea879f0 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/5_eval_5_a69fcdfd49e2066116b6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a69fcdfd49e2066116b64ed866b1feec0677babe9575a14c121d0519d8c7ddc6 +size 1233195 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/6_eval_6_ae418a6c99a51b4201d6.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/6_eval_6_ae418a6c99a51b4201d6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9aa2e666c5ee3808df8759634fa5d7268ebfb951 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/6_eval_6_ae418a6c99a51b4201d6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae418a6c99a51b4201d6c6588f7cb94c4106062b3f5a598d62fb3d4c361fe692 +size 1234700 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/7_eval_7_f02cea2dbceefebb3b97.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/7_eval_7_f02cea2dbceefebb3b97.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e92cdd03054010ee56db87f7277e6eeea20c0b79 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/7_eval_7_f02cea2dbceefebb3b97.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f02cea2dbceefebb3b97fe2cabe09bff692847bbfd980a30122dc91aeeff2e88 +size 1252184 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/8_eval_8_fb8f14c1c353cc80983a.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/8_eval_8_fb8f14c1c353cc80983a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9b8e2a4591a0ad2fa9ad824ea5d462f970f62dac --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/8_eval_8_fb8f14c1c353cc80983a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb8f14c1c353cc80983ac758cdb4a327ec227033c4ac10db233eaab9871e1ff0 +size 1248692 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/9_eval_9_19795df8831a10c10837.mp4 b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/9_eval_9_19795df8831a10c10837.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e27fb42dbe3ba64595aa8590aa9222196db401b9 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/9_eval_9_19795df8831a10c10837.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19795df8831a10c108372bc2039f39d52ab4951b1f2751808168cafd03ed2c78 +size 835686 diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..61f12900d262b22ae663ff435fc70a53c1da092a --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log @@ -0,0 +1,1126 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736995541.678756]: resetting environment +[INFO] [1736995541.684369]: cleared current plan +[INFO] [1736995541.685631]: moving to home +[INFO] [1736995542.687782]: reached home +[INFO] [1736995542.688979]: reset action history +[INFO] [1736995545.447720]: environment reset +Starting episode 0 +[INFO] [1736995545.449606]: resetting environment +[INFO] [1736995545.454307]: cleared current plan +[INFO] [1736995545.455622]: moving to home +[INFO] [1736995546.457282]: reached home +[INFO] [1736995546.459603]: reset action history +[INFO] [1736995549.166734]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736995559.517638]: resetting environment +[INFO] [1736995559.521030]: cleared current plan +[INFO] [1736995559.522071]: moving to home +[INFO] [1736995560.524321]: reached home +[INFO] [1736995560.525564]: reset action history +[INFO] [1736995563.233122]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_0_2025-01-15-21-46-03.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:38951/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2471003] +process[print_text-2]: started with pid [2471027] +process[rosbag_record-3]: started with pid [2471028] +started bagging! +preempting episode +[INFO] [1736995577.466284]: Storing episode... +[rosbag_record-3] killing on exit[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736995578.221426]: Stored episode 1. +[INFO] [1736995578.222313]: Saving video... +[INFO] [1736995578.427409]: Video saved! +Starting episode 1 +[INFO] [1736995578.437485]: resetting environment +[INFO] [1736995578.450638]: cleared current plan +[INFO] [1736995578.450911]: moving to home +[INFO] [1736995581.953712]: reached home +[INFO] [1736995581.954179]: reset action history +[INFO] [1736995584.662777]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736995598.705920]: resetting environment +[INFO] [1736995598.708618]: cleared current plan +[INFO] [1736995598.708771]: moving to home +[INFO] [1736995599.709374]: reached home +[INFO] [1736995599.709812]: reset action history +[INFO] [1736995602.416801]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_1_2025-01-15-21-46-42.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:40357/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2471134] +process[print_text-5]: started with pid [2471135] +process[rosbag_record-6]: started with pid [2471159] +started bagging! +[INFO] [1736995629.384917]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1736995630.130836]: Stored episode 2. +[INFO] [1736995630.131117]: Saving video... +[INFO] [1736995630.403086]: Video saved! +Starting episode 2 +[INFO] [1736995630.436019]: resetting environment +[INFO] [1736995630.446857]: cleared current plan +[INFO] [1736995630.447077]: moving to home +[INFO] [1736995634.049207]: reached home +[INFO] [1736995634.050716]: reset action history +[INFO] [1736995636.832322]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736995646.600836]: resetting environment +[INFO] [1736995646.607909]: cleared current plan +[INFO] [1736995646.609303]: moving to home +[INFO] [1736995647.613169]: reached home +[INFO] [1736995647.613711]: reset action history +[INFO] [1736995650.323524]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_2_2025-01-15-21-47-30.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:37253/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2471300] +process[print_text-8]: started with pid [2471301] +process[rosbag_record-9]: started with pid [2471302] +started bagging! +preempting episode +[INFO] [1736995666.729858]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1736995667.477016]: Stored episode 3. +[INFO] [1736995667.479134]: Saving video... +[INFO] [1736995668.083610]: Video saved! +Starting episode 3 +[INFO] [1736995668.116157]: resetting environment +[INFO] [1736995668.140842]: cleared current plan +[INFO] [1736995668.142029]: moving to home +[WARN] [1736995667.979041]: Plan exhausted +[WARN] [1736995668.019938]: Plan exhausted +[WARN] [1736995668.063128]: Plan exhausted +[WARN] [1736995668.099876]: Plan exhausted +[WARN] [1736995668.138935]: Plan exhausted +[INFO] [1736995671.747090]: reached home +[INFO] [1736995671.749142]: reset action history +[INFO] [1736995674.671000]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736995675.840930]: resetting environment +[INFO] [1736995675.854748]: cleared current plan +[INFO] [1736995675.857574]: moving to home +[INFO] [1736995676.862777]: reached home +[INFO] [1736995676.863518]: reset action history +[INFO] [1736995679.641886]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_3_2025-01-15-21-47-59.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:42669/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2471417] +process[print_text-11]: started with pid [2471418] +process[rosbag_record-12]: started with pid [2471419] +started bagging! +[INFO] [1736995706.600881]: Storing episode... +[print_text-11] killing on exit[pub_text-10] killing on exit +[rosbag_record-12] killing on exit +[INFO] [1736995707.385338]: Stored episode 4. +[INFO] [1736995707.386458]: Saving video... +[INFO] [1736995707.946784]: Video saved! +Starting episode 4 +[INFO] [1736995708.006771]: resetting environment +[INFO] [1736995708.017957]: cleared current plan +[INFO] [1736995708.018258]: moving to home +[WARN] [1736995707.769669]: Plan exhausted +[WARN] [1736995707.807204]: Plan exhausted +[WARN] [1736995707.846833]: Plan exhausted +[WARN] [1736995707.886678]: Plan exhausted +[WARN] [1736995707.929400]: Plan exhausted +[WARN] [1736995707.967686]: Plan exhausted +[WARN] [1736995708.009303]: Plan exhausted +[INFO] [1736995711.625091]: reached home +[INFO] [1736995711.626271]: reset action history +[INFO] [1736995714.415201]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736995721.459133]: resetting environment +[INFO] [1736995721.473588]: cleared current plan +[INFO] [1736995721.475142]: moving to home +[INFO] [1736995722.479970]: reached home +[INFO] [1736995722.487409]: reset action history +[INFO] [1736995725.264311]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_4_2025-01-15-21-48-45.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:39249/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2471585] +process[print_text-14]: started with pid [2471586] +process[rosbag_record-15]: started with pid [2471587] +started bagging! +preempting episode +[INFO] [1736995743.100989]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736995743.856288]: Stored episode 5. +[INFO] [1736995743.858950]: Saving video... +[INFO] [1736995744.209862]: Video saved! +Starting episode 5 +[INFO] [1736995744.261248]: resetting environment +[INFO] [1736995744.282911]: cleared current plan +[INFO] [1736995744.285484]: moving to home +[INFO] [1736995747.895752]: reached home +[INFO] [1736995747.902945]: reset action history +[INFO] [1736995750.692277]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736995753.136023]: resetting environment +[INFO] [1736995753.155797]: cleared current plan +[INFO] [1736995753.158604]: moving to home +[INFO] [1736995754.162842]: reached home +[INFO] [1736995754.170127]: reset action history +[INFO] [1736995756.961179]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_5_2025-01-15-21-49-16.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:37151/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-16]: started with pid [2471716] +process[print_text-17]: started with pid [2471717] +process[rosbag_record-18]: started with pid [2471718] +started bagging! +[INFO] [1736995785.699619]: Storing episode... +[rosbag_record-18] killing on exit +[pub_text-16] killing on exit +[print_text-17] killing on exit +[INFO] [1736995786.494123]: Stored episode 6. +[INFO] [1736995786.514804]: Saving video... +[INFO] [1736995786.975965]: Video saved! +Starting episode 6 +[INFO] [1736995787.058951]: resetting environment +[INFO] [1736995787.091126]: cleared current plan +[INFO] [1736995787.096065]: moving to home +[WARN] [1736995786.853600]: Plan exhausted +[WARN] [1736995786.886468]: Plan exhausted +[WARN] [1736995786.926574]: Plan exhausted +[WARN] [1736995786.969269]: Plan exhausted +[WARN] [1736995787.008866]: Plan exhausted +[WARN] [1736995787.047649]: Plan exhausted +[WARN] [1736995787.086492]: Plan exhausted +[INFO] [1736995790.706547]: reached home +[INFO] [1736995790.710476]: reset action history +[INFO] [1736995793.575642]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736995800.580036]: resetting environment +[INFO] [1736995800.586695]: cleared current plan +[INFO] [1736995800.586865]: moving to home +[INFO] [1736995801.588949]: reached home +[INFO] [1736995801.592330]: reset action history +[INFO] [1736995804.377644]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_6_2025-01-15-21-50-04.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:46051/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-19]: started with pid [2471860] +process[print_text-20]: started with pid [2471861] +process[rosbag_record-21]: started with pid [2471862] +started bagging! +For topic gripper_width: timestamp difference is -115705370 for nearest: 1736995822787250400 - target: 1736995822902955770 at idx 247 +[INFO] [1736995833.356687]: Storing episode... +[rosbag_record-21] killing on exit +[print_text-20] killing on exit[pub_text-19] killing on exit +[INFO] [1736995834.149137]: Stored episode 7. +[INFO] [1736995834.152597]: Saving video... +[INFO] [1736995834.376736]: Video saved! +Starting episode 7 +[INFO] [1736995834.440859]: resetting environment +[INFO] [1736995834.476942]: cleared current plan +[INFO] [1736995834.483719]: moving to home +[INFO] [1736995838.112038]: reached home +[INFO] [1736995838.139867]: reset action history +[INFO] [1736995841.026516]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736995857.131668]: resetting environment +[INFO] [1736995857.156901]: cleared current plan +[INFO] [1736995857.163894]: moving to home +[INFO] [1736995858.167172]: reached home +[INFO] [1736995858.178484]: reset action history +[INFO] [1736995861.005569]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_7_2025-01-15-21-51-01.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:33131/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-22]: started with pid [2471991] +process[print_text-23]: started with pid [2471992] +process[rosbag_record-24]: started with pid [2471994] +started bagging! +For topic gripper_width: timestamp difference is -158670948 for nearest: 1736995888787249043 - target: 1736995888945919991 at idx 249 +[INFO] [1736995890.227692]: Storing episode... +[print_text-23] killing on exit +[rosbag_record-24] killing on exit +[pub_text-22] killing on exit +[INFO] [1736995891.074424]: Stored episode 8. +[INFO] [1736995891.089222]: Saving video... +[INFO] [1736995891.353086]: Video saved! +Starting episode 8 +[INFO] [1736995891.454679]: resetting environment +[INFO] [1736995891.486791]: cleared current plan +[INFO] [1736995891.490528]: moving to home +[INFO] [1736995895.097685]: reached home +[INFO] [1736995895.101672]: reset action history +[INFO] [1736995897.980860]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736995901.913386]: resetting environment +[INFO] [1736995901.927061]: cleared current plan +[INFO] [1736995901.927444]: moving to home +[INFO] [1736995902.931927]: reached home +[INFO] [1736995902.943531]: reset action history +[INFO] [1736995905.820961]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_8_2025-01-15-21-51-45.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:40385/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-25]: started with pid [2472138] +process[print_text-26]: started with pid [2472139] +process[rosbag_record-27]: started with pid [2472140] +started bagging! +For topic gripper_width: timestamp difference is 103972877 for nearest: 1736995921453938688 - target: 1736995921349965811 at idx 211 +For topic gripper_width: timestamp difference is 101882508 for nearest: 1736995927853949597 - target: 1736995927752067089 at idx 249 +For topic gripper_width: timestamp difference is 109714937 for nearest: 1736995931653938246 - target: 1736995931544223309 at idx 249 +[INFO] [1736995935.422808]: Storing episode... +[rosbag_record-27] killing on exit +[print_text-26] killing on exit +[pub_text-25] killing on exit +[INFO] [1736995936.182910]: Stored episode 9. +[INFO] [1736995936.194174]: Saving video... +[INFO] [1736995936.750515]: Video saved! +Starting episode 9 +[INFO] [1736995936.845321]: resetting environment +[INFO] [1736995936.872074]: cleared current plan +[INFO] [1736995936.875252]: moving to home +[WARN] [1736995936.766456]: Plan exhausted +[WARN] [1736995936.810717]: Plan exhausted +[WARN] [1736995936.847439]: Plan exhausted +[INFO] [1736995940.484844]: reached home +[INFO] [1736995940.496794]: reset action history +[INFO] [1736995943.425445]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736995951.585867]: resetting environment +[INFO] [1736995951.622193]: cleared current plan +[INFO] [1736995951.633713]: moving to home +[INFO] [1736995952.645905]: reached home +[INFO] [1736995952.649597]: reset action history +[INFO] [1736995955.559502]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_9_2025-01-15-21-52-35.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:43979/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-28]: started with pid [2472283] +process[print_text-29]: started with pid [2472284] +process[rosbag_record-30]: started with pid [2472285] +started bagging! +preempting episode +[INFO] [1736995971.947357]: Storing episode... +[print_text-29] killing on exit +[rosbag_record-30] killing on exit +[pub_text-28] killing on exit +[INFO] [1736995972.770325]: Stored episode 10. +[INFO] [1736995972.782634]: Saving video... +[INFO] [1736995973.250473]: Video saved! +Starting episode 10 +[INFO] [1736995973.363045]: resetting environment +[INFO] [1736995973.401555]: cleared current plan +[INFO] [1736995973.408286]: moving to home +[WARN] [1736995973.223962]: Plan exhausted +[WARN] [1736995973.256768]: Plan exhausted +[WARN] [1736995973.288930]: Plan exhausted +[WARN] [1736995973.327149]: Plan exhausted +[WARN] [1736995973.368382]: Plan exhausted +[INFO] [1736995977.035413]: reached home +[INFO] [1736995977.061774]: reset action history +[INFO] [1736995979.994861]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736995982.071408]: resetting environment +[INFO] [1736995982.105691]: cleared current plan +[INFO] [1736995982.124222]: moving to home +[INFO] [1736995983.129390]: reached home +[INFO] [1736995983.132440]: reset action history +[INFO] [1736995986.091481]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_10_2025-01-15-21-53-06.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:45361/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-31]: started with pid [2472415] +process[print_text-32]: started with pid [2472416] +process[rosbag_record-33]: started with pid [2472417] +started bagging! +For topic gripper_width: timestamp difference is -110811390 for nearest: 1736995995787252508 - target: 1736995995898063898 at idx 140 +For topic gripper_width: timestamp difference is -101655165 for nearest: 1736996005587208351 - target: 1736996005688863516 at idx 249 +For topic gripper_width: timestamp difference is -102114859 for nearest: 1736996005987280187 - target: 1736996006089395046 at idx 249 +preempting episode +[INFO] [1736996006.870170]: Storing episode... +[print_text-32] killing on exit +[pub_text-31] killing on exit +[rosbag_record-33] killing on exit +[INFO] [1736996007.703775]: Stored episode 11. +[INFO] [1736996007.707549]: Saving video... +[INFO] [1736996008.300448]: Video saved! +Starting episode 11 +[INFO] [1736996008.419210]: resetting environment +[INFO] [1736996008.453961]: cleared current plan +[INFO] [1736996008.460983]: moving to home +[WARN] [1736996008.126476]: Plan exhausted +[WARN] [1736996008.166895]: Plan exhausted +[WARN] [1736996008.212223]: Plan exhausted +[WARN] [1736996008.251291]: Plan exhausted +[WARN] [1736996008.288230]: Plan exhausted +[WARN] [1736996008.329525]: Plan exhausted +[WARN] [1736996008.366660]: Plan exhausted +[WARN] [1736996008.407113]: Plan exhausted +[WARN] [1736996008.447109]: Plan exhausted +[INFO] [1736996012.086724]: reached home +[INFO] [1736996012.094949]: reset action history +[INFO] [1736996015.089758]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736996016.493389]: resetting environment +[INFO] [1736996016.538603]: cleared current plan +[INFO] [1736996016.539790]: moving to home +[INFO] [1736996017.555078]: reached home +[INFO] [1736996017.573089]: reset action history +[INFO] [1736996020.492662]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_11_2025-01-15-21-53-40.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:39447/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-34]: started with pid [2472558] +process[print_text-35]: started with pid [2472559] +process[rosbag_record-36]: started with pid [2472560] +started bagging! +preempting episode +[INFO] [1736996042.675710]: Storing episode... +[rosbag_record-36] killing on exit +[pub_text-34] killing on exit +[print_text-35] killing on exit +[INFO] [1736996043.443901]: Stored episode 12. +[INFO] [1736996043.446287]: Saving video... +[INFO] [1736996043.938275]: Video saved! +Starting episode 12 +[INFO] [1736996044.074125]: resetting environment +[INFO] [1736996044.091713]: cleared current plan +[INFO] [1736996044.099430]: moving to home +[WARN] [1736996043.967648]: Plan exhausted +[WARN] [1736996044.006613]: Plan exhausted +[WARN] [1736996044.046468]: Plan exhausted +[WARN] [1736996044.086625]: Plan exhausted +[INFO] [1736996047.711222]: reached home +[INFO] [1736996047.716497]: reset action history +[INFO] [1736996050.808357]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736996059.129936]: resetting environment +[INFO] [1736996059.177989]: cleared current plan +[INFO] [1736996059.188546]: moving to home +[INFO] [1736996060.198414]: reached home +[INFO] [1736996060.212644]: reset action history +[INFO] [1736996063.177852]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_12_2025-01-15-21-54-23.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:41633/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-37]: started with pid [2472667] +process[print_text-38]: started with pid [2472668] +process[rosbag_record-39]: started with pid [2472692] +started bagging! +[INFO] [1736996090.202324]: Storing episode... +[rosbag_record-39] killing on exit +[pub_text-37] killing on exit +[print_text-38] killing on exit +[INFO] [1736996091.053067]: Stored episode 13. +[INFO] [1736996091.065577]: Saving video... +[INFO] [1736996091.659123]: Video saved! +[WARN] [1736996091.366671]: Plan exhausted +[WARN] [1736996091.412679]: Plan exhausted +[WARN] [1736996091.446480]: Plan exhausted +[WARN] [1736996091.491329]: Plan exhausted +[WARN] [1736996091.528861]: Plan exhausted +[WARN] [1736996091.567091]: Plan exhausted +[WARN] [1736996091.606907]: Plan exhausted +[WARN] [1736996091.646580]: Plan exhausted +[WARN] [1736996091.688119]: Plan exhausted +[WARN] [1736996091.726447]: Plan exhausted +[WARN] [1736996091.769763]: Plan exhausted +[WARN] [1736996091.808749]: Plan exhausted +Starting episode 13 +[INFO] [1736996091.796232]: resetting environment +[INFO] [1736996091.844279]: cleared current plan +[INFO] [1736996091.845109]: moving to home +[INFO] [1736996095.454351]: reached home +[INFO] [1736996095.463279]: reset action history +[INFO] [1736996098.470592]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736996101.631875]: resetting environment +[INFO] [1736996101.695683]: cleared current plan +[INFO] [1736996101.709305]: moving to home +[INFO] [1736996102.729236]: reached home +[INFO] [1736996102.737239]: reset action history +[INFO] [1736996105.717882]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_13_2025-01-15-21-55-05.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:46171/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-40]: started with pid [2472834] +process[print_text-41]: started with pid [2472835] +process[rosbag_record-42]: started with pid [2472836] +started bagging! +For topic gripper_width: timestamp difference is -100548089 for nearest: 1736996121787270009 - target: 1736996121887818098 at idx 229 +For topic gripper_width: timestamp difference is 102805682 for nearest: 1736996126187249966 - target: 1736996126084444284 at idx 249 +For topic gripper_width: timestamp difference is 104326193 for nearest: 1736996127987246220 - target: 1736996127882920027 at idx 249 +For topic gripper_width: timestamp difference is 105035031 for nearest: 1736996128787272418 - target: 1736996128682237387 at idx 249 +[INFO] [1736996139.363731]: Storing episode... +[rosbag_record-42] killing on exit +[print_text-41] killing on exit +[pub_text-40] killing on exit +[INFO] [1736996140.133277]: Stored episode 14. +[INFO] [1736996140.154699]: Saving video... +[INFO] [1736996140.800308]: Video saved! +Starting episode 14 +[INFO] [1736996140.960318]: resetting environment +[INFO] [1736996141.028986]: cleared current plan +[INFO] [1736996141.029851]: moving to home +[WARN] [1736996140.731272]: Plan exhausted +[WARN] [1736996140.768726]: Plan exhausted +[WARN] [1736996140.817865]: Plan exhausted +[WARN] [1736996140.855957]: Plan exhausted +[WARN] [1736996140.887847]: Plan exhausted +[WARN] [1736996140.928354]: Plan exhausted +[WARN] [1736996140.967510]: Plan exhausted +[WARN] [1736996141.019354]: Plan exhausted +[INFO] [1736996144.751359]: reached home +[INFO] [1736996144.762895]: reset action history +[INFO] [1736996147.956968]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736996159.173964]: resetting environment +[INFO] [1736996159.222208]: cleared current plan +[INFO] [1736996159.271412]: moving to home +[INFO] [1736996160.288758]: reached home +[INFO] [1736996160.312546]: reset action history +[INFO] [1736996163.438273]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_14_2025-01-15-21-56-03.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:36177/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-43]: started with pid [2472957] +process[print_text-44]: started with pid [2472981] +process[rosbag_record-45]: started with pid [2472982] +started bagging! +For topic gripper_width: timestamp difference is 106266616 for nearest: 1736996171687246678 - target: 1736996171580980062 at idx 111 +For topic gripper_width: timestamp difference is 112148227 for nearest: 1736996178887244643 - target: 1736996178775096416 at idx 190 +For topic gripper_width: timestamp difference is 113009733 for nearest: 1736996179887271446 - target: 1736996179774261713 at idx 198 +For topic gripper_width: timestamp difference is -117069859 for nearest: 1736996183687266450 - target: 1736996183804336309 at idx 248 +[INFO] [1736996190.472265]: Storing episode... +[rosbag_record-45] killing on exit[print_text-44] killing on exit +[pub_text-43] killing on exit +[INFO] [1736996191.291342]: Stored episode 15. +[INFO] [1736996191.316166]: Saving video... +[WARN] [1736996191.821310]: Plan exhausted +[WARN] [1736996191.854446]: Plan exhausted +[WARN] [1736996191.887163]: Plan exhausted +[WARN] [1736996191.927459]: Plan exhausted +[WARN] [1736996191.978480]: Plan exhausted +[INFO] [1736996191.845312]: Video saved! +Starting episode 15 +[INFO] [1736996192.023173]: resetting environment +[INFO] [1736996192.068438]: cleared current plan +[INFO] [1736996192.087517]: moving to home +[WARN] [1736996192.045211]: Plan exhausted +[WARN] [1736996192.055684]: Plan exhausted +[INFO] [1736996195.813225]: reached home +[INFO] [1736996195.833047]: reset action history +[INFO] [1736996199.044901]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736996401.914503]: resetting environment +[INFO] [1736996402.027784]: cleared current plan +[INFO] [1736996402.036371]: moving to home +[INFO] [1736996403.059597]: reached home +[INFO] [1736996403.072315]: reset action history +[INFO] [1736996406.214572]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_15_2025-01-15-22-00-06.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:38843/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-46]: started with pid [2473464] +process[print_text-47]: started with pid [2473465] +process[rosbag_record-48]: started with pid [2473466] +started bagging! +preempting episode +[INFO] [1736996427.685771]: Storing episode... +[rosbag_record-48] killing on exit[print_text-47] killing on exit +[pub_text-46] killing on exit +[INFO] [1736996429.188924]: Stored episode 16. +[INFO] [1736996429.198371]: Saving video... +[INFO] [1736996429.916719]: Video saved! +Starting episode 16 +[INFO] [1736996430.037865]: resetting environment +[WARN] [1736996428.931432]: Plan exhausted +[WARN] [1736996428.966451]: Plan exhausted +[WARN] [1736996429.010812]: Plan exhausted +[WARN] [1736996429.050025]: Plan exhausted +[WARN] [1736996429.107583]: Plan exhausted +[WARN] [1736996429.129773]: Plan exhausted +[WARN] [1736996429.169383]: Plan exhausted +[WARN] [1736996429.206834]: Plan exhausted +[WARN] [1736996429.252803]: Plan exhausted +[WARN] [1736996429.295743]: Plan exhausted +[WARN] [1736996429.326442]: Plan exhausted +[WARN] [1736996429.372940]: Plan exhausted +[WARN] [1736996429.415522]: Plan exhausted +[WARN] [1736996429.452301]: Plan exhausted +[WARN] [1736996429.495564]: Plan exhausted +[WARN] [1736996429.528426]: Plan exhausted +[WARN] [1736996429.588033]: Plan exhausted +[WARN] [1736996429.619671]: Plan exhausted +[WARN] [1736996429.666970]: Plan exhausted +[WARN] [1736996429.686712]: Plan exhausted +[WARN] [1736996429.735851]: Plan exhausted +[WARN] [1736996429.767290]: Plan exhausted +[WARN] [1736996429.808096]: Plan exhausted +[WARN] [1736996429.846460]: Plan exhausted +[WARN] [1736996429.886476]: Plan exhausted +[WARN] [1736996429.927824]: Plan exhausted +[WARN] [1736996429.966812]: Plan exhausted +[WARN] [1736996430.010747]: Plan exhausted +[WARN] [1736996430.046416]: Plan exhausted +[INFO] [1736996430.063974]: cleared current plan +[INFO] [1736996430.064231]: moving to home +[INFO] [1736996433.569360]: reached home +[INFO] [1736996433.584130]: reset action history +[INFO] [1736996436.749343]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736996444.775977]: resetting environment +[INFO] [1736996444.860404]: cleared current plan +[INFO] [1736996444.892098]: moving to home +[INFO] [1736996445.906982]: reached home +[INFO] [1736996445.935156]: reset action history +[INFO] [1736996449.141279]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_16_2025-01-15-22-00-49.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:32949/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-49]: started with pid [2473607] +process[print_text-50]: started with pid [2473608] +process[rosbag_record-51]: started with pid [2473609] +started bagging! +[INFO] [1736996485.001498]: Storing episode... +[print_text-50] killing on exit +[rosbag_record-51] killing on exit +[pub_text-49] killing on exit +[INFO] [1736996485.812369]: Stored episode 17. +[INFO] [1736996485.838877]: Saving video... +[WARN] [1736996486.150568]: Plan exhausted +[WARN] [1736996486.195855]: Plan exhausted +[WARN] [1736996486.236134]: Plan exhausted +[WARN] [1736996486.247707]: Plan exhausted +[INFO] [1736996486.140785]: Video saved! +Starting episode 17 +[INFO] [1736996486.313881]: resetting environment +[INFO] [1736996486.368413]: cleared current plan +[INFO] [1736996486.382154]: moving to home +[WARN] [1736996486.287313]: Plan exhausted +[WARN] [1736996486.332886]: Plan exhausted +[INFO] [1736996490.019059]: reached home +[INFO] [1736996490.053253]: reset action history +[INFO] [1736996493.149350]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736996534.938113]: resetting environment +[INFO] [1736996535.055061]: cleared current plan +[INFO] [1736996535.072962]: moving to home +[INFO] [1736996536.084827]: reached home +[INFO] [1736996536.098946]: reset action history +[INFO] [1736996539.204233]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_17_2025-01-15-22-02-19.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:40405/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-52]: started with pid [2473799] +process[print_text-53]: started with pid [2473800] +process[rosbag_record-54]: started with pid [2473824] +started bagging! +preempting episode +[INFO] [1736996553.713692]: Storing episode... +[pub_text-52] killing on exit +[print_text-53] killing on exit +[rosbag_record-54] killing on exit +[INFO] [1736996554.560978]: Stored episode 18. +[INFO] [1736996554.590851]: Saving video... +[INFO] [1736996555.269187]: Video saved! +Starting episode 18 +[INFO] [1736996555.439517]: resetting environment +[INFO] [1736996555.480904]: cleared current plan +[INFO] [1736996555.487741]: moving to home +[WARN] [1736996555.007196]: Plan exhausted +[WARN] [1736996555.048359]: Plan exhausted +[WARN] [1736996555.087008]: Plan exhausted +[WARN] [1736996555.128984]: Plan exhausted +[WARN] [1736996555.168090]: Plan exhausted +[WARN] [1736996555.206735]: Plan exhausted +[WARN] [1736996555.247750]: Plan exhausted +[WARN] [1736996555.297911]: Plan exhausted +[WARN] [1736996555.332031]: Plan exhausted +[WARN] [1736996555.367558]: Plan exhausted +[WARN] [1736996555.410092]: Plan exhausted +[WARN] [1736996555.453810]: Plan exhausted +[INFO] [1736996559.105060]: reached home +[INFO] [1736996559.124220]: reset action history +[INFO] [1736996562.290601]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736996578.312407]: resetting environment +[INFO] [1736996578.349948]: cleared current plan +[INFO] [1736996578.351228]: moving to home +[INFO] [1736996579.357263]: reached home +[INFO] [1736996579.374511]: reset action history +[INFO] [1736996582.366336]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_18_2025-01-15-22-03-02.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:45545/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-55]: started with pid [2473973] +process[print_text-56]: started with pid [2473974] +process[rosbag_record-57]: started with pid [2473975] +started bagging! +[INFO] [1736996616.843411]: Storing episode... +[rosbag_record-57] killing on exit +[print_text-56] killing on exit[pub_text-55] killing on exit +[INFO] [1736996617.678285]: Stored episode 19. +[INFO] [1736996617.688725]: Saving video... +[WARN] [1736996618.170385]: Plan exhausted +[WARN] [1736996618.206715]: Plan exhausted +[WARN] [1736996618.247437]: Plan exhausted +[WARN] [1736996618.286681]: Plan exhausted +[WARN] [1736996618.329539]: Plan exhausted +[WARN] [1736996618.366473]: Plan exhausted +[WARN] [1736996618.410442]: Plan exhausted +[INFO] [1736996618.310089]: Video saved! +Starting episode 19 +[INFO] [1736996618.543163]: resetting environment +[INFO] [1736996618.618332]: cleared current plan +[INFO] [1736996618.638966]: moving to home +[WARN] [1736996618.446755]: Plan exhausted +[WARN] [1736996618.486455]: Plan exhausted +[WARN] [1736996618.530676]: Plan exhausted +[WARN] [1736996618.567127]: Plan exhausted +[WARN] [1736996618.607780]: Plan exhausted +[INFO] [1736996622.278625]: reached home +[INFO] [1736996622.315572]: reset action history +[INFO] [1736996625.688889]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736996633.275081]: resetting environment +[INFO] [1736996633.378494]: cleared current plan +[INFO] [1736996633.413970]: moving to home +[INFO] [1736996634.454348]: reached home +[INFO] [1736996634.479449]: reset action history +[INFO] [1736996637.706177]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/episode_rosbags/episode_19_2025-01-15-22-03-57.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2470707.log +started roslaunch server http://158.130.50.37:45497/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-58]: started with pid [2474120] +process[print_text-59]: started with pid [2474121] +process[rosbag_record-60]: started with pid [2474122] +started bagging! +[INFO] [1736996676.588781]: Storing episode... +[pub_text-58] killing on exit +[rosbag_record-60] killing on exit +[print_text-59] killing on exit +[INFO] [1736996677.435383]: Stored episode 20. +[INFO] [1736996677.467200]: Saving video... +[INFO] [1736996678.203218]: Video saved! +[WARN] [1736996677.953486]: Plan exhausted +[WARN] [1736996677.978631]: Plan exhausted +[WARN] [1736996678.018129]: Plan exhausted +[WARN] [1736996678.046537]: Plan exhausted +[WARN] [1736996678.086466]: Plan exhausted +[WARN] [1736996678.128153]: Plan exhausted +[WARN] [1736996678.166924]: Plan exhausted +[WARN] [1736996678.228153]: Plan exhausted +[WARN] [1736996678.259347]: Plan exhausted +[WARN] [1736996678.295814]: Plan exhausted +[WARN] [1736996678.329496]: Plan exhausted +[WARN] [1736996678.428035]: Plan exhausted +[WARN] [1736996678.441299]: Plan exhausted +[WARN] [1736996678.446523]: Plan exhausted +[INFO] [1736996678.377690]: resetting environment +[INFO] [1736996678.463964]: cleared current plan +[INFO] [1736996678.477533]: moving to home +[INFO] [1736996682.203296]: reached home +[INFO] [1736996682.210418]: reset action history +[INFO] [1736996685.577942]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/requirements.txt b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch new file mode 100644 index 0000000000000000000000000000000000000000..a480ed1cd3b38df5c4e7ecff218642b42e0995b6 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch @@ -0,0 +1,91 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..2659f59 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 5 ++num_eval: 20 + + debug_timestamps: False + open_loop: False +@@ -133,14 +133,14 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-# wandb_run_id: '1002_0' ++wandb_run_id: '3456_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..134df65 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,12 +167,14 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ '3456_0', ++ '3456_1', + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-metadata.json b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b26d7b98d8885de7c6bb81f6d004e4c392a1a9a0 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-16T02:45:13.004292", + "startedAt": "2025-01-16T02:45:12.616933", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "ec42957dc10d08dbd8b134558fcd8f9e470ccc31" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3748.5323750000002, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.919, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 326.04469299316406 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f550cf08fc2e16c11e8258b0c192bbb23f8683f6 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "2eb6cf19687a4fabfe3c6f8e612266c1da2cc0076d60b65057514a926a74144f", "size": 660308, "path": "media/videos/eval/0_eval_0_2eb6cf19687a4fabfe3c.mp4"}, "global_step": 2195, "_timestamp": 1736996696.777449, "_runtime": 1184.152766942978, "_step": 41, "eval/1_eval": {"_type": "video-file", "sha256": "b5d7870b7b04a6ef485f99d3b1ba052460f021a02fc0a4bcb269c4a104c53731", "size": 1158221, "path": "media/videos/eval/1_eval_1_b5d7870b7b04a6ef485f.mp4"}, "eval/num_success": 12.0, "episode": 40.0, "eval/success_rate": 0.6000000238418579, "eval/success_rate_lb_0.01": 0.32809674739837646, "eval/success_rate_ub_0.01": 0.8194341659545898, "eval/success_rate_lb_0.025": 0.3644341826438904, "eval/success_rate_ub_0.025": 0.7783663868904114, "eval/success_rate_lb_0.05": 0.4294946491718292, "eval/success_rate_ub_0.05": 0.7635830044746399, "eval/success_rate_lb_0.1": 0.4492693543434143, "eval/success_rate_ub_0.1": 0.7487341165542603, "eval/2_eval": {"_type": "video-file", "sha256": "f98c31cd2e05324cffd88658a9b1b179814eff0305d38aba54e59921246de742", "size": 810213, "path": "media/videos/eval/2_eval_2_f98c31cd2e05324cffd8.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "89821fb09b372630838b8abd67ec78c34acf566da143b01acc2d08888bb0bc6c", "size": 1153488, "path": "media/videos/eval/3_eval_3_89821fb09b372630838b.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "9fb52356d0a213033cfe25b61a1934cbe1ee32f73cd0b718905313f2644136c0", "size": 897581, "path": "media/videos/eval/4_eval_4_9fb52356d0a213033cfe.mp4"}, "eval/5_eval": {"_type": "video-file", "sha256": "a69fcdfd49e2066116b64ed866b1feec0677babe9575a14c121d0519d8c7ddc6", "size": 1233195, "path": "media/videos/eval/5_eval_5_a69fcdfd49e2066116b6.mp4"}, "eval/6_eval": {"_type": "video-file", "sha256": "ae418a6c99a51b4201d6c6588f7cb94c4106062b3f5a598d62fb3d4c361fe692", "size": 1234700, "path": "media/videos/eval/6_eval_6_ae418a6c99a51b4201d6.mp4"}, "eval/7_eval": {"_type": "video-file", "sha256": "f02cea2dbceefebb3b97fe2cabe09bff692847bbfd980a30122dc91aeeff2e88", "size": 1252184, "path": "media/videos/eval/7_eval_7_f02cea2dbceefebb3b97.mp4"}, "eval/8_eval": {"_type": "video-file", "sha256": "fb8f14c1c353cc80983ac758cdb4a327ec227033c4ac10db233eaab9871e1ff0", "size": 1248692, "path": "media/videos/eval/8_eval_8_fb8f14c1c353cc80983a.mp4"}, "eval/9_eval": {"_type": "video-file", "sha256": "19795df8831a10c108372bc2039f39d52ab4951b1f2751808168cafd03ed2c78", "size": 835686, "path": "media/videos/eval/9_eval_9_19795df8831a10c10837.mp4"}, "eval/10_eval": {"_type": "video-file", "sha256": "6e43fe2801f3a905e582de81a9680045e1a4067f03042a5ff19f96803aa6f698", "size": 1031684, "path": "media/videos/eval/10_eval_10_6e43fe2801f3a905e582.mp4"}, "eval/11_eval": {"_type": "video-file", "sha256": "4263c7dc48b07a1c552cc54d84f88093f2e788ee860e0a2d14e44c2cc1fdaeef", "size": 1084332, "path": "media/videos/eval/11_eval_11_4263c7dc48b07a1c552c.mp4"}, "eval/12_eval": {"_type": "video-file", "sha256": "241be90b3f1c54884e79ee6f66e49e2b9cd981f6a46f20431ce6ffa92227c5b2", "size": 1167206, "path": "media/videos/eval/12_eval_12_241be90b3f1c54884e79.mp4"}, "eval/13_eval": {"_type": "video-file", "sha256": "cb3449f02cfc7cf55b8d5c8b2858c5450855681d80b52de97d006761f8ae751b", "size": 1419465, "path": "media/videos/eval/13_eval_13_cb3449f02cfc7cf55b8d.mp4"}, "eval/14_eval": {"_type": "video-file", "sha256": "074bdfc80a4a9d765f986d8266f086e36f6336cbfc94d03d20f82b5898467960", "size": 1166864, "path": "media/videos/eval/14_eval_14_074bdfc80a4a9d765f98.mp4"}, "eval/15_eval": {"_type": "video-file", "sha256": "76fa4e81e9e73fef9cf0cbf0cae5de5fadd030e8577b710e5fbbc635395fbbad", "size": 961267, "path": "media/videos/eval/15_eval_15_76fa4e81e9e73fef9cf0.mp4"}, "eval/16_eval": {"_type": "video-file", "sha256": "5019757f76965e10782b8dbf76cb0ec5dd9466a48a2c73f37eaae45a01df3902", "size": 1505807, "path": "media/videos/eval/16_eval_16_5019757f76965e10782b.mp4"}, "eval/17_eval": {"_type": "video-file", "sha256": "2f96b121fd6f99c60674db7dbf14916a988cd7103feb371d1024b26d7d81e965", "size": 680421, "path": "media/videos/eval/17_eval_17_2f96b121fd6f99c60674.mp4"}, "eval/18_eval": {"_type": "video-file", "sha256": "c8c84019c8dc8ba0e4ead689259df8c2ee18d6119c191be15ecdb4cd11fc2f0f", "size": 1441508, "path": "media/videos/eval/18_eval_18_c8c84019c8dc8ba0e4ea.mp4"}, "eval/19_eval": {"_type": "video-file", "sha256": "3f737e19fed74dd298b263bf46de6d1133a6a7db350c8d12a1fdd73af013c15b", "size": 1623338, "path": "media/videos/eval/19_eval_19_3f737e19fed74dd298b2.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "0f278608fd5017506fe4f527ec27d547e17ec2d76458cccd0632f86bb42c70dd", "size": 398, "artifact_path": "wandb-client-artifact://gmglk0qfncfveewkcnr5f840an8ep6q3k4hp1dk09pr161vaom2kwi1ac7wu1e89s5av87t9qyvrzb5tmd0o2ki2y4gebsg62z7oiewi4dwil1k9cgmgg1xoef2uf7h1/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://9f1ah09fcgeutd5ejddq7xjzznckxead7okd97loadrwj7ilda6e9fkgw32ikfeqk4wy3e58spkbxtggd5a67jo4bbvbrlsi1rqol09147s520fnwyk5xtet6peesl35:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_20_0f278608fd5017506fe4.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 27, "total_num_episodes": 40, "total_success_rate": 0.675, "total_success_rate_lb_0.01": 0.4954639971265328, "total_success_rate_ub_0.01": 0.827632218533793, "total_success_rate_lb_0.025": 0.5256678312965555, "total_success_rate_ub_0.025": 0.7974396570922111, "total_success_rate_lb_0.05": 0.558896869409226, "total_success_rate_ub_0.05": 0.7784236967006781, "total_success_rate_lb_0.1": 0.5626946240538138, "total_success_rate_ub_0.1": 0.7666594795349383, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "ee38b3327b46c4981cc4a92f810c1069311f022bbd167828aba1a49f72d24a75", "size": 505, "artifact_path": "wandb-client-artifact://nmkciqug3v35vwznetllp32s3c18wx4lcqcrltuxguhm532ocap9mz94qetp09mu0ree2evavclzojm1rlw41k8v1vd66fyhm67ic24h81dfq2xe71vpvjgdq0bpvdxl/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://jcydnh7dcadq2lswi9jrgafz1xhi6eixfszatyns6do1wfkqy8ht638jre839tnpueq0sb908o75dvskvqfku4tvj8d4kw85wlzs1pdqycub7mcxbilrm6uqcwzvthri:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_21_ee38b3327b46c4981cc4.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 1201}, "eval/total_success_rate": 0.675000011920929, "eval/total_success_rate_lb_0.01": 0.4954639971256256, "eval/total_success_rate_ub_0.01": 0.8276321887969971, "eval/total_success_rate_lb_0.025": 0.5256678462028503, "eval/total_success_rate_ub_0.025": 0.7974396347999573, "eval/total_success_rate_lb_0.05": 0.5588968396186829, "eval/total_success_rate_ub_0.05": 0.7784236669540405, "eval/total_success_rate_lb_0.1": 0.5626946091651917, "eval/total_success_rate_ub_0.1": 0.7666594982147217} \ No newline at end of file diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/logs/debug-internal.log b/roatk2bk/wandb/run-20250115_214512-roatk2bk/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3a88dba77f6e2088adb58db153e8ffea84ee5770 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/logs/debug-internal.log @@ -0,0 +1,2430 @@ +2025-01-15 21:45:12,624 INFO StreamThr :2470873 [internal.py:wandb_internal():85] W&B internal server running at pid: 2470873, started at: 2025-01-15 21:45:12.624488 +2025-01-15 21:45:12,625 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status +2025-01-15 21:45:12,637 INFO WriterThread:2470873 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/run-roatk2bk.wandb +2025-01-15 21:45:12,638 DEBUG SenderThread:2470873 [sender.py:send():379] send: header +2025-01-15 21:45:12,645 DEBUG SenderThread:2470873 [sender.py:send():379] send: run +2025-01-15 21:45:12,893 INFO SenderThread:2470873 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files +2025-01-15 21:45:12,893 INFO SenderThread:2470873 [sender.py:_start_run_threads():1188] run started: roatk2bk with start time 1736995512.624682 +2025-01-15 21:45:12,912 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: check_version +2025-01-15 21:45:12,912 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: check_version +2025-01-15 21:45:12,986 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: run_start +2025-01-15 21:45:12,990 DEBUG HandlerThread:2470873 [system_info.py:__init__():26] System info init +2025-01-15 21:45:12,991 DEBUG HandlerThread:2470873 [system_info.py:__init__():41] System info init done +2025-01-15 21:45:12,991 INFO HandlerThread:2470873 [system_monitor.py:start():194] Starting system monitor +2025-01-15 21:45:12,991 INFO SystemMonitor:2470873 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-15 21:45:12,991 INFO HandlerThread:2470873 [system_monitor.py:probe():214] Collecting system info +2025-01-15 21:45:12,991 INFO SystemMonitor:2470873 [interfaces.py:start():188] Started cpu monitoring +2025-01-15 21:45:12,992 INFO SystemMonitor:2470873 [interfaces.py:start():188] Started disk monitoring +2025-01-15 21:45:12,992 INFO SystemMonitor:2470873 [interfaces.py:start():188] Started gpu monitoring +2025-01-15 21:45:12,993 INFO SystemMonitor:2470873 [interfaces.py:start():188] Started memory monitoring +2025-01-15 21:45:12,993 INFO SystemMonitor:2470873 [interfaces.py:start():188] Started network monitoring +2025-01-15 21:45:13,004 DEBUG HandlerThread:2470873 [system_info.py:probe():152] Probing system +2025-01-15 21:45:13,005 DEBUG HandlerThread:2470873 [system_info.py:_probe_git():137] Probing git +2025-01-15 21:45:13,010 DEBUG HandlerThread:2470873 [system_info.py:_probe_git():145] Probing git done +2025-01-15 21:45:13,010 DEBUG HandlerThread:2470873 [system_info.py:probe():200] Probing system done +2025-01-15 21:45:13,010 DEBUG HandlerThread:2470873 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-16T02:45:13.004292', 'startedAt': '2025-01-16T02:45:12.616933', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.5323750000002, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.919, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 326.04469299316406}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-15 21:45:13,010 INFO HandlerThread:2470873 [system_monitor.py:probe():224] Finished collecting system info +2025-01-15 21:45:13,010 INFO HandlerThread:2470873 [system_monitor.py:probe():227] Publishing system info +2025-01-15 21:45:13,010 DEBUG HandlerThread:2470873 [system_info.py:_save_code():44] Saving code +2025-01-15 21:45:13,011 DEBUG HandlerThread:2470873 [system_info.py:_save_code():67] Saving code done +2025-01-15 21:45:13,011 DEBUG HandlerThread:2470873 [system_info.py:_save_patches():84] Saving git patches +2025-01-15 21:45:13,109 DEBUG HandlerThread:2470873 [system_info.py:_save_patches():126] Saving git patches done +2025-01-15 21:45:13,109 INFO HandlerThread:2470873 [system_monitor.py:probe():229] Finished publishing system info +2025-01-15 21:45:13,113 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:45:13,113 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-15 21:45:13,113 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-15 21:45:13,113 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-15 21:45:13,113 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-15 21:45:13,359 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: python_packages +2025-01-15 21:45:13,359 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: python_packages +2025-01-15 21:45:13,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:45:13,360 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:45:13,363 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:13,363 INFO HandlerThread:2470873 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/tb" + save: true +} +control { + relay_id: "7e40abce871040128ee2222932bf4fd5" +} +_info { + stream_id: "roatk2bk" +} + +2025-01-15 21:45:13,364 DEBUG HandlerThread:2470873 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-15 21:45:13,367 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/2krs8t93-wandb-metadata.json +2025-01-15 21:45:13,395 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/i5dtzjt2-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-15 21:45:13,398 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/5i0rcxno-diff.patch +2025-01-15 21:45:13,404 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/zzqthulp-code/FISH/eval_robot.py +2025-01-15 21:45:13,535 DEBUG SenderThread:2470873 [sender.py:send():379] send: telemetry +2025-01-15 21:45:13,536 DEBUG SenderThread:2470873 [sender.py:send():379] send: tbrecord +2025-01-15 21:45:13,536 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:45:13,536 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 with policy live +2025-01-15 21:45:13,712 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/2kz91b1o-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:45:13,895 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:45:13,895 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-15 21:45:13,895 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/requirements.txt +2025-01-15 21:45:13,895 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/diff.patch +2025-01-15 21:45:13,895 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:45:13,895 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-metadata.json +2025-01-15 21:45:13,895 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/code/FISH/eval_robot.py +2025-01-15 21:45:13,895 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/code/FISH +2025-01-15 21:45:13,896 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/code +2025-01-15 21:45:14,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:15,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:15,894 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:45:16,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:17,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:18,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:45:18,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:19,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:20,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:21,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:22,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:23,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:45:23,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:24,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:25,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:26,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:27,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:28,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:45:28,360 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:45:28,405 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:28,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:45:29,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:30,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:31,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:32,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:33,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:34,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:45:34,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:35,363 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:36,363 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:37,363 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:38,363 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:39,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:45:39,363 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:40,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:41,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:41,678 DEBUG SenderThread:2470873 [sender.py:send():379] send: config +2025-01-15 21:45:42,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:43,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:45:43,360 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:45:43,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:43,901 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:45:44,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:44,435 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:45:44,902 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/config.yaml +2025-01-15 21:45:45,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:46,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:47,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:47,903 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:45:48,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:49,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:50,210 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:45:50,365 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:51,365 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:51,904 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:45:52,365 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:53,365 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:54,365 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:55,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:45:55,365 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:56,365 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:57,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:58,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:45:58,360 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:45:58,405 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:45:59,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:00,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:00,524 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:01,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:01,907 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:46:02,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:03,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:04,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:05,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:05,527 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:05,908 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:46:06,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:07,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:08,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:09,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:10,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:11,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:11,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:12,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:12,993 DEBUG SystemMonitor:2470873 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-15 21:46:12,994 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:46:13,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:46:13,360 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:46:13,401 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:14,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:15,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:16,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:16,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:17,368 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:18,368 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:18,469 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:46:18,470 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:46:18,470 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_2eb6cf19687a4fabfe3c.mp4 with policy now +2025-01-15 21:46:18,470 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:46:18,470 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:46:18,471 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:46:18,768 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/z7jz6cz9-media/videos/eval/0_eval_0_2eb6cf19687a4fabfe3c.mp4 +2025-01-15 21:46:18,912 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/0_eval_0_2eb6cf19687a4fabfe3c.mp4 +2025-01-15 21:46:18,912 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:46:18,912 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media +2025-01-15 21:46:18,912 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos +2025-01-15 21:46:18,912 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:46:19,189 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/o7263fbs-media/videos/eval/0_eval_0_2eb6cf19687a4fabfe3c.mp4 +2025-01-15 21:46:19,369 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:19,912 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:46:20,368 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:21,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:21,954 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:22,369 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:23,369 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:23,914 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:46:24,369 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:25,369 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:25,914 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:46:26,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:27,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:27,706 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:28,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:46:28,360 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:46:28,401 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:29,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:30,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:31,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:32,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:33,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:33,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:34,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:35,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:36,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:37,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:38,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:38,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:38,920 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:46:39,179 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/6brbwr5a-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:46:39,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:39,920 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:46:40,372 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:41,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:41,921 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:46:42,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:43,003 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:46:43,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:46:43,361 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:46:43,372 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:43,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:43,921 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:46:44,372 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:45,372 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:46,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:47,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:48,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:49,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:49,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:50,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:51,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:52,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:53,374 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:54,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:46:54,374 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:55,374 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:56,374 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:57,374 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:58,361 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:46:58,361 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:46:58,405 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:59,374 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:46:59,420 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:00,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:01,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:02,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:03,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:04,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:05,363 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:05,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:06,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:07,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:08,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:09,376 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:10,376 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:10,404 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:10,445 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:47:10,446 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:47:10,446 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b5d7870b7b04a6ef485f.mp4 with policy now +2025-01-15 21:47:10,446 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:47:10,446 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:47:10,447 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:47:10,708 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/bgeeyww1-media/videos/eval/1_eval_1_b5d7870b7b04a6ef485f.mp4 +2025-01-15 21:47:10,930 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:47:10,930 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/1_eval_1_b5d7870b7b04a6ef485f.mp4 +2025-01-15 21:47:10,930 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:47:11,202 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/kgz7uqes-media/videos/eval/1_eval_1_b5d7870b7b04a6ef485f.mp4 +2025-01-15 21:47:11,377 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:11,930 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:12,377 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:12,995 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:47:13,368 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:47:13,368 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:47:13,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:14,380 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:15,385 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:15,932 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:16,093 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:16,385 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:17,381 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:17,932 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:18,380 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:19,380 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:20,381 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:21,369 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:21,381 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:22,381 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:23,381 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:24,381 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:25,381 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:26,381 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:26,588 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:26,936 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:47:27,382 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:27,810 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:47:27,811 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:47:27,811 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:47:27,936 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:27,936 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:47:28,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:47:28,363 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:47:28,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:29,382 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:29,937 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:30,382 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:31,382 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:31,723 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:31,937 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:32,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:33,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:34,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:35,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:36,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:37,368 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:37,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:38,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:39,384 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:40,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:41,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:42,369 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:42,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:42,995 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:47:43,362 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:47:43,362 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:47:43,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:44,384 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:45,384 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:46,384 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:47,386 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:47,478 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:47,943 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:48,113 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:47:48,113 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_f98c31cd2e05324cffd8.mp4 with policy now +2025-01-15 21:47:48,113 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:47:48,114 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:47:48,114 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:47:48,115 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:47:48,354 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/22gfzdwj-media/videos/eval/2_eval_2_f98c31cd2e05324cffd8.mp4 +2025-01-15 21:47:48,389 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:48,944 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:47:48,944 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/2_eval_2_f98c31cd2e05324cffd8.mp4 +2025-01-15 21:47:48,944 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:47:49,217 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/yjg1aet9-media/videos/eval/2_eval_2_f98c31cd2e05324cffd8.mp4 +2025-01-15 21:47:49,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:49,944 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:50,389 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:50,945 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:51,390 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:52,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:52,793 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:53,397 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:53,946 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:54,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:55,393 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:55,947 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:47:55,947 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:56,200 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/mlrje92h-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:47:56,388 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:56,617 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:47:56,617 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:47:56,617 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:47:56,947 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:47:57,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:57,905 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:47:57,947 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:47:58,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:47:58,367 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:47:58,413 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:47:59,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:00,390 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:01,390 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:01,949 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:48:02,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:03,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:03,390 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:04,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:05,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:06,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:07,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:08,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:08,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:09,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:10,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:11,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:12,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:12,996 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:48:13,363 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:48:13,363 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:48:13,405 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:13,485 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:14,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:15,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:16,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:17,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:18,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:19,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:19,393 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:20,393 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:21,393 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:22,393 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:23,393 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:24,371 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:24,393 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:25,393 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:26,393 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:27,395 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:27,958 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:48:27,996 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:48:27,996 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_89821fb09b372630838b.mp4 with policy now +2025-01-15 21:48:28,005 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:48:28,006 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:48:28,006 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:48:28,006 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:48:28,254 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/b0rvfr7r-media/videos/eval/3_eval_3_89821fb09b372630838b.mp4 +2025-01-15 21:48:28,363 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:48:28,363 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:48:28,394 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:28,958 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:48:28,959 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/3_eval_3_89821fb09b372630838b.mp4 +2025-01-15 21:48:28,959 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:48:29,255 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/rjnbpuoi-media/videos/eval/3_eval_3_89821fb09b372630838b.mp4 +2025-01-15 21:48:29,398 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:29,419 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:29,959 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:48:30,403 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:30,959 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:48:31,403 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:32,403 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:33,427 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:33,960 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:48:34,403 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:34,457 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:35,405 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:35,961 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:48:36,402 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:37,405 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:38,411 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:39,404 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:40,382 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:40,420 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:41,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:41,672 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:48:41,672 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:48:41,673 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:48:41,963 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:48:41,963 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:48:42,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:42,996 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:48:43,366 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:48:43,367 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:48:43,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:43,964 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:48:44,410 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:45,411 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:45,964 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:48:46,313 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:46,416 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:47,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:47,965 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:48:48,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:49,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:50,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:51,372 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:51,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:52,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:53,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:54,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:55,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:56,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:48:56,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:57,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:58,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:48:58,364 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:48:58,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:48:59,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:00,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:01,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:01,442 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:02,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:03,416 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:03,971 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:04,245 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:49:04,245 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_9fb52356d0a213033cfe.mp4 with policy now +2025-01-15 21:49:04,254 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:49:04,255 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:49:04,255 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:49:04,255 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:49:04,411 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:04,483 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/6v0b62sc-media/videos/eval/4_eval_4_9fb52356d0a213033cfe.mp4 +2025-01-15 21:49:04,971 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:49:04,971 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/4_eval_4_9fb52356d0a213033cfe.mp4 +2025-01-15 21:49:04,971 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:49:05,243 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/9pexg8i7-media/videos/eval/4_eval_4_9fb52356d0a213033cfe.mp4 +2025-01-15 21:49:05,416 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:05,971 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:06,411 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:07,286 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:07,419 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:08,422 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:09,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:09,973 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:10,422 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:11,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:11,974 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:12,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:12,737 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:12,997 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:49:13,369 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:49:13,369 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:49:13,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:13,713 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:49:13,713 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:49:13,713 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:49:13,974 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:49:13,975 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:13,975 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:49:14,254 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/3wz60emi-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:49:14,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:15,421 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:15,975 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:16,421 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:17,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:17,976 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:18,005 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:18,435 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:19,421 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:19,977 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:20,421 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:21,421 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:22,421 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:23,375 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:23,421 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:24,421 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:25,422 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:26,422 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:27,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:28,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:49:28,365 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:49:28,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:28,428 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:29,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:30,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:31,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:32,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:33,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:34,376 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:34,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:35,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:36,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:37,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:38,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:39,376 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:39,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:40,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:41,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:42,425 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:42,997 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:49:43,364 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:49:43,365 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:49:43,425 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:44,425 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:44,510 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:45,425 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:46,435 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:47,038 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:49:47,038 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_a69fcdfd49e2066116b6.mp4 with policy now +2025-01-15 21:49:47,042 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:49:47,042 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:49:47,042 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:49:47,042 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:49:47,335 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/mhfqktbw-media/videos/eval/5_eval_5_a69fcdfd49e2066116b6.mp4 +2025-01-15 21:49:47,429 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:47,987 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:47,987 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:49:47,987 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/5_eval_5_a69fcdfd49e2066116b6.mp4 +2025-01-15 21:49:47,987 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:49:48,334 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/xj0rb057-media/videos/eval/5_eval_5_a69fcdfd49e2066116b6.mp4 +2025-01-15 21:49:48,441 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:48,987 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:49,440 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:50,405 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:50,439 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:51,439 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:51,988 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:52,438 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:53,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:54,440 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:55,445 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:55,617 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:49:55,990 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:49:56,442 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:57,445 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:58,378 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:49:58,379 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:49:58,449 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:49:59,444 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:00,453 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:00,771 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:50:00,771 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:50:00,772 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:00,772 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:50:00,991 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:50:00,992 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:50:01,444 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:01,992 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:02,445 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:03,446 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:03,992 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:04,446 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:05,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:05,993 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:06,382 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:06,446 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:07,446 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:07,994 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:08,446 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:09,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:10,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:11,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:12,379 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:12,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:12,998 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:50:13,365 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:50:13,365 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:50:13,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:14,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:15,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:16,447 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:17,444 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:17,448 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:18,449 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:19,449 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:20,449 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:21,449 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:22,450 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:22,987 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:23,450 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:24,000 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:24,450 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:25,451 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:26,451 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:27,451 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:28,365 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:50:28,365 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:50:28,451 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:28,504 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:29,451 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:30,451 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:31,451 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:32,451 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:33,453 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:34,004 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:34,149 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:34,435 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:50:34,435 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_ae418a6c99a51b4201d6.mp4 with policy now +2025-01-15 21:50:34,439 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:50:34,439 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:50:34,439 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:50:34,440 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:50:34,461 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:34,690 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/56yq0pm1-media/videos/eval/6_eval_6_ae418a6c99a51b4201d6.mp4 +2025-01-15 21:50:35,004 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:50:35,004 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/6_eval_6_ae418a6c99a51b4201d6.mp4 +2025-01-15 21:50:35,004 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:50:35,285 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/i8416a62-media/videos/eval/6_eval_6_ae418a6c99a51b4201d6.mp4 +2025-01-15 21:50:35,462 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:36,004 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:36,463 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:37,461 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:38,468 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:39,398 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:39,481 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:40,006 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:40,483 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:41,466 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:42,007 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:42,469 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:42,998 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:50:43,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:50:43,370 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:50:43,485 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:44,445 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:44,472 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:45,493 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:46,485 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:47,489 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:48,488 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:49,491 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:50,398 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:50,489 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:51,494 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:52,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:53,490 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:54,495 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:55,399 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:50:55,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:56,498 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:57,491 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:57,842 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:50:57,842 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:50:57,842 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:50:58,012 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:50:58,012 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:50:58,013 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:50:58,263 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/b1z9epwb-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:50:58,385 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:50:58,385 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:50:58,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:50:59,494 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:00,013 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:00,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:00,536 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:01,495 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:02,014 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:02,502 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:03,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:04,014 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:04,496 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:05,496 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:05,541 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:06,015 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:06,496 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:07,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:08,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:09,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:10,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:11,396 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:11,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:12,498 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:12,998 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:51:13,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:51:13,374 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:51:13,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:14,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:15,498 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:16,498 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:16,517 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:17,498 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:18,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:19,498 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:20,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:21,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:22,397 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:22,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:23,500 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:24,500 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:25,500 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:26,500 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:27,397 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:27,500 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:28,374 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:51:28,374 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:51:28,500 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:29,500 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:30,024 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:30,514 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:31,430 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:51:31,430 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_f02cea2dbceefebb3b97.mp4 with policy now +2025-01-15 21:51:31,446 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:51:31,446 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:51:31,446 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:51:31,447 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:51:31,504 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:31,676 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/hgjwvamf-media/videos/eval/7_eval_7_f02cea2dbceefebb3b97.mp4 +2025-01-15 21:51:32,025 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:32,025 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:51:32,025 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/7_eval_7_f02cea2dbceefebb3b97.mp4 +2025-01-15 21:51:32,025 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:51:32,315 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/dxfpwmx0-media/videos/eval/7_eval_7_f02cea2dbceefebb3b97.mp4 +2025-01-15 21:51:32,492 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:32,520 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:33,520 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:34,026 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:34,525 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:35,522 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:36,027 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:36,526 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:37,526 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:37,982 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:38,541 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:39,529 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:40,029 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:40,541 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:41,543 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:41,897 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:51:41,897 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:51:41,897 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:51:42,030 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:51:42,030 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:51:42,551 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:42,999 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:51:42,999 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:43,382 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:51:43,383 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:51:43,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:44,030 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:44,543 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:45,543 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:46,549 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:47,550 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:48,032 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:48,387 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:48,548 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:49,544 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:50,033 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:51:50,544 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:51,544 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:52,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:53,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:53,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:54,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:55,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:56,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:57,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:58,378 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:51:58,378 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:51:58,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:51:58,549 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:51:59,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:00,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:01,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:02,037 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:02,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:03,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:04,410 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:04,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:05,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:06,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:07,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:08,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:09,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:10,041 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:10,410 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:10,547 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:11,547 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:12,547 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:12,999 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:52:13,378 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:52:13,378 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:52:13,547 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:14,042 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:14,547 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:15,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:15,547 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:16,043 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:16,555 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:16,815 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:52:16,815 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_fb8f14c1c353cc80983a.mp4 with policy now +2025-01-15 21:52:16,829 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:52:16,829 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:52:16,829 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:52:16,830 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:52:17,044 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:52:17,044 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/8_eval_8_fb8f14c1c353cc80983a.mp4 +2025-01-15 21:52:17,044 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:52:17,075 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/xsm8xo6m-media/videos/eval/8_eval_8_fb8f14c1c353cc80983a.mp4 +2025-01-15 21:52:17,344 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/ctcy9ilg-media/videos/eval/8_eval_8_fb8f14c1c353cc80983a.mp4 +2025-01-15 21:52:17,561 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:18,044 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:18,575 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:19,044 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:19,561 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:20,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:20,555 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:21,554 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:22,045 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:22,569 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:23,556 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:24,046 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:24,570 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:25,477 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:25,564 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:26,578 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:27,565 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:28,399 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:52:28,399 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:52:28,587 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:29,572 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:30,483 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:30,584 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:31,576 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:31,960 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:52:31,960 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:52:31,961 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:52:32,049 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:52:32,050 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:32,050 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:52:32,302 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/hj7yr4f7-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:52:32,578 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:33,581 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:34,050 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:34,586 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:35,561 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:35,605 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:36,577 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:37,577 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:38,052 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:38,577 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:39,577 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:40,577 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:41,416 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:41,577 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:42,577 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:43,000 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:52:43,379 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:52:43,379 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:52:43,578 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:44,578 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:45,578 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:46,468 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:46,578 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:47,578 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:48,578 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:49,578 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:50,579 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:51,579 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:51,928 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:52,590 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:53,349 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:52:53,349 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_19795df8831a10c10837.mp4 with policy now +2025-01-15 21:52:53,355 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:52:53,356 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:52:53,356 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:52:53,356 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:52:53,583 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/e9k3rryo-media/videos/eval/9_eval_9_19795df8831a10c10837.mp4 +2025-01-15 21:52:53,585 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:54,058 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:54,058 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:52:54,058 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/9_eval_9_19795df8831a10c10837.mp4 +2025-01-15 21:52:54,059 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:52:54,340 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/61hoxzze-media/videos/eval/9_eval_9_19795df8831a10c10837.mp4 +2025-01-15 21:52:54,592 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:55,058 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:55,595 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:56,597 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:57,047 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:52:57,625 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:58,060 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:52:58,399 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:52:58,400 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:52:58,622 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:52:59,600 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:00,617 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:01,618 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:02,058 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:02,061 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:53:02,061 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:02,612 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:03,000 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:53:03,000 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:53:03,001 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:53:03,061 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:53:03,062 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:53:03,629 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:04,062 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:04,634 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:05,619 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:06,611 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:07,141 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:07,651 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:08,063 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:08,623 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:09,630 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:10,643 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:11,623 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:12,065 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:12,623 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:13,000 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:53:13,002 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:13,379 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:53:13,380 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:53:13,623 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:14,623 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:15,623 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:16,623 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:17,623 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:18,017 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:18,067 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:18,624 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:19,624 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:20,624 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:21,624 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:22,624 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:23,422 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:23,624 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:24,624 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:25,624 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:26,624 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:27,635 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:28,071 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:28,381 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:53:28,381 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_6e43fe2801f3a905e582.mp4 with policy now +2025-01-15 21:53:28,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:53:28,391 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:53:28,391 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:53:28,392 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:53:28,397 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:53:28,397 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:53:28,526 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:28,629 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/jp7098hy-media/videos/eval/10_eval_10_6e43fe2801f3a905e582.mp4 +2025-01-15 21:53:28,630 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:29,072 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:53:29,072 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/10_eval_10_6e43fe2801f3a905e582.mp4 +2025-01-15 21:53:29,072 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:53:29,342 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/utk77r6u-media/videos/eval/10_eval_10_6e43fe2801f3a905e582.mp4 +2025-01-15 21:53:29,646 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:30,072 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:30,634 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:31,073 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:31,666 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:32,642 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:33,647 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:34,074 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:34,432 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:34,649 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:35,656 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:36,075 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:36,657 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:37,044 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:53:37,044 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:53:37,045 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:53:37,075 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:53:37,075 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:53:37,325 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/9thvpq01-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:53:37,668 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:38,075 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:38,659 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:39,458 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:39,662 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:40,660 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:41,672 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:42,077 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:42,672 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:43,001 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:53:43,391 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:53:43,391 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:53:43,667 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:44,515 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:44,683 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:45,681 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:46,664 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:47,664 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:48,080 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:53:48,664 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:49,664 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:50,455 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:50,664 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:51,664 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:52,665 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:53,665 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:54,665 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:55,665 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:56,455 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:53:56,665 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:57,665 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:58,383 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:53:58,383 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:53:58,665 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:53:59,666 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:00,666 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:01,500 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:01,666 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:02,709 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:03,692 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:04,043 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:54:04,043 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_4263c7dc48b07a1c552c.mp4 with policy now +2025-01-15 21:54:04,061 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:54:04,062 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:54:04,062 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:54:04,062 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:54:04,086 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:04,086 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:54:04,086 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/11_eval_11_4263c7dc48b07a1c552c.mp4 +2025-01-15 21:54:04,086 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:54:04,323 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/0alu68rb-media/videos/eval/11_eval_11_4263c7dc48b07a1c552c.mp4 +2025-01-15 21:54:04,586 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/wr03doaa-media/videos/eval/11_eval_11_4263c7dc48b07a1c552c.mp4 +2025-01-15 21:54:04,683 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:05,678 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:06,087 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:06,696 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:07,087 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:07,103 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:07,697 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:08,680 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:09,697 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:10,088 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:10,701 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:11,693 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:12,089 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:12,697 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:12,853 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:13,001 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:54:13,399 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:54:13,399 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:54:13,707 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:14,697 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:15,694 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:16,693 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:17,727 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:18,480 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:18,709 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:19,091 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:54:19,096 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:54:19,096 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:54:19,100 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:54:19,730 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:20,092 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:54:20,092 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:20,092 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:54:20,727 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:21,718 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:22,093 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:22,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:23,513 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:23,710 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:24,094 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:24,710 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:25,710 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:26,711 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:27,711 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:28,384 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:54:28,384 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:54:28,712 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:29,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:29,712 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:30,712 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:31,712 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:32,712 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:33,712 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:34,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:35,468 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:35,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:36,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:37,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:38,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:39,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:40,468 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:40,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:41,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:42,714 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:43,002 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:54:43,384 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:54:43,385 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:54:43,714 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:44,714 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:45,489 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:45,714 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:46,714 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:47,714 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:48,714 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:49,714 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:50,729 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:51,053 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:51,725 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:51,773 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:54:51,773 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:54:51,773 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_241be90b3f1c54884e79.mp4 with policy now +2025-01-15 21:54:51,774 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:54:51,774 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:54:51,774 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:54:52,030 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/lmglli64-media/videos/eval/12_eval_12_241be90b3f1c54884e79.mp4 +2025-01-15 21:54:52,105 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:54:52,105 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/12_eval_12_241be90b3f1c54884e79.mp4 +2025-01-15 21:54:52,105 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:54:52,401 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/r6bjjpye-media/videos/eval/12_eval_12_241be90b3f1c54884e79.mp4 +2025-01-15 21:54:52,735 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:53,105 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:53,734 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:54,750 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:55,106 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:55,763 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:56,505 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:54:56,757 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:57,107 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:54:57,755 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:58,401 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:54:58,401 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:54:58,751 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:54:59,763 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:00,761 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:01,108 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:01,522 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:01,776 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:02,109 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:55:02,149 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:55:02,150 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:55:02,150 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:55:02,787 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:03,110 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:03,110 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:55:03,788 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:04,766 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:05,111 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:05,789 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:06,765 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:06,804 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:07,803 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:08,791 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:09,112 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:09,800 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:10,793 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:11,807 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:12,179 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:12,791 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:13,002 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:55:13,388 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:55:13,388 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:55:13,791 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:14,791 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:15,115 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:15,792 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:16,791 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:17,532 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:17,792 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:18,792 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:19,792 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:20,792 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:21,792 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:22,792 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:23,029 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:23,792 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:24,793 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:25,119 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:25,793 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:26,793 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:27,793 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:28,037 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:28,387 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:55:28,387 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:55:28,793 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:29,121 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:29,793 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:30,793 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:31,122 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:31,794 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:32,794 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:33,479 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:33,794 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:34,794 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:35,794 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:36,795 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:37,795 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:38,479 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:38,795 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:39,799 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:40,823 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:40,926 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:55:40,926 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_cb3449f02cfc7cf55b8d.mp4 with policy now +2025-01-15 21:55:40,935 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:55:40,935 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:55:40,936 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:55:40,936 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:55:41,126 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:41,126 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:55:41,126 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/13_eval_13_cb3449f02cfc7cf55b8d.mp4 +2025-01-15 21:55:41,126 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:55:41,174 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/lgjmpjp7-media/videos/eval/13_eval_13_cb3449f02cfc7cf55b8d.mp4 +2025-01-15 21:55:41,428 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/tx0smn3m-media/videos/eval/13_eval_13_cb3449f02cfc7cf55b8d.mp4 +2025-01-15 21:55:41,827 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:42,807 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:43,003 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:55:43,126 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:43,413 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:55:43,413 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:55:43,512 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:43,829 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:44,831 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:45,820 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:46,822 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:47,128 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:47,824 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:48,522 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:48,846 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:49,886 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:50,853 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:51,130 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:55:51,882 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:52,850 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:53,885 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:54,507 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:55:54,860 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:55,889 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:56,868 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:57,877 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:58,435 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:55:58,435 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:55:58,881 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:55:59,133 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:55:59,219 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:55:59,219 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:55:59,219 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:55:59,375 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/t0pun5g6-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:55:59,880 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:00,133 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 21:56:00,133 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:56:00,274 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:00,888 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:01,134 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:01,871 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:02,906 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:03,134 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:03,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:04,873 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:05,136 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:05,753 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:05,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:06,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:07,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:08,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:09,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:10,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:11,488 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:11,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:12,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:13,005 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:56:13,139 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:13,401 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:56:13,401 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:56:13,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:14,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:15,875 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:16,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:16,875 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:17,875 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:18,875 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:19,875 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:20,875 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:21,142 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:21,876 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:21,933 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:22,876 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:23,143 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:23,876 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:24,876 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:25,876 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:26,876 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:27,145 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:27,876 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:27,926 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:28,401 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:56:28,401 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:56:28,876 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:29,876 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:30,898 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:31,888 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:31,978 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 21:56:31,978 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_074bdfc80a4a9d765f98.mp4 with policy now +2025-01-15 21:56:31,991 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:56:31,991 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 21:56:31,991 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:56:31,992 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:56:32,147 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 21:56:32,148 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/14_eval_14_074bdfc80a4a9d765f98.mp4 +2025-01-15 21:56:32,148 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 21:56:32,225 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/tyvt3rah-media/videos/eval/14_eval_14_074bdfc80a4a9d765f98.mp4 +2025-01-15 21:56:32,492 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/hvflza3a-media/videos/eval/14_eval_14_074bdfc80a4a9d765f98.mp4 +2025-01-15 21:56:32,891 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:33,088 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:33,148 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:33,890 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:34,912 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:35,149 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:35,908 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:36,914 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:37,939 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:38,515 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:38,925 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:39,151 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:39,936 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:40,917 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:41,152 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 21:56:41,940 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:42,966 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:43,003 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:56:43,404 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:56:43,404 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:56:43,960 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:44,508 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:44,954 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:45,942 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:46,938 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:47,950 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:48,943 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:49,517 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:49,935 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:50,940 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:51,978 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:52,951 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:53,960 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:54,537 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:56:54,962 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:55,969 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:56,988 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:57,973 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:56:58,427 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:56:58,428 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:56:58,979 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:00,018 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:00,528 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:01,023 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:02,025 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:02,992 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:03,987 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:04,997 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:05,537 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:05,988 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:07,004 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:08,017 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:09,027 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:10,024 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:11,027 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:11,524 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:12,040 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:13,004 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:57:13,026 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:13,419 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:57:13,420 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:57:14,035 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:15,023 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:16,020 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:17,022 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:17,523 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:18,030 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:19,024 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:20,074 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:21,067 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:22,043 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:22,526 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:23,053 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:24,055 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:25,059 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:26,064 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:27,050 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:27,554 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:28,069 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:28,432 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:57:28,432 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:57:29,077 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:30,066 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:31,064 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:32,083 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:32,556 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:33,096 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:34,070 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:35,094 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:36,072 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:37,109 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:38,089 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:38,520 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:39,105 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:40,102 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:41,095 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:42,099 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:43,004 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:57:43,126 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:43,426 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:57:43,426 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:57:43,539 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:44,111 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:45,127 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:46,118 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:47,118 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:48,122 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:49,113 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:49,526 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:50,125 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:51,123 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:52,120 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:53,150 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:54,153 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:54,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:57:55,156 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:56,145 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:57,134 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:58,160 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:58,425 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:57:58,425 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:57:59,155 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:57:59,547 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:00,146 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:01,182 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:02,151 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:03,167 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:04,180 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:04,555 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:05,170 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:06,172 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:07,179 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:08,174 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:09,194 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:10,232 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:10,546 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:11,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:12,205 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:13,005 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:58:13,210 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:13,453 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:58:13,453 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:58:14,246 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:15,245 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:15,559 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:16,265 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:17,226 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:18,234 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:19,234 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:20,234 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:20,601 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:21,239 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:22,237 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:23,242 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:24,244 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:25,273 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:26,262 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:26,571 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:27,270 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:28,251 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:28,431 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:58:28,431 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:58:29,264 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:30,285 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:31,278 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:31,580 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:32,280 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:33,283 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:34,308 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:35,318 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:36,352 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:37,309 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:37,579 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:38,309 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:39,343 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:40,313 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:41,311 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:42,321 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:42,640 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:43,005 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:58:43,329 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:43,464 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:58:43,464 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:58:44,347 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:45,339 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:46,342 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:47,342 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:48,339 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:48,596 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:49,378 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:50,358 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:51,359 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:52,345 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:53,358 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:53,604 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:54,354 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:55,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:56,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:57,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:58,372 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:58:58,448 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:58:58,448 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:58:58,645 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:58:59,373 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:00,378 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:01,372 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:02,401 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:03,401 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:04,402 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:04,608 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:05,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:06,406 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:07,426 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:08,431 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:09,417 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:10,429 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:10,608 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:11,424 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:12,431 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:13,006 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:59:13,451 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:13,463 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:59:13,463 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:59:14,465 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:15,454 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:15,619 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:16,428 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:17,456 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:18,457 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:19,455 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:20,461 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:20,621 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:21,448 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:22,487 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:23,495 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:24,496 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:25,452 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:26,458 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:26,614 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:27,481 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:28,453 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:59:28,453 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:59:28,497 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:29,475 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:30,461 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:31,478 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:31,645 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:32,474 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:33,469 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:34,490 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:35,530 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:36,512 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:37,505 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:37,623 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:38,526 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:39,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:40,531 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:41,545 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:42,563 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:42,642 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:43,006 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 21:59:43,461 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:59:43,461 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:59:43,566 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:44,557 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:45,558 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:46,561 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:47,565 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:48,595 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:48,626 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:49,569 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:50,575 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:51,589 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:52,588 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:53,583 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:54,590 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:54,626 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:59:55,586 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:56,602 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:57,591 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:58,461 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:59:58,461 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:59:58,589 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:59,605 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:59:59,643 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:00,616 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:01,609 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:02,237 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:00:02,498 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:00:02,498 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:00:02,498 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:00:02,617 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:03,238 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:03,238 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:00:03,614 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:04,633 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:04,644 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:05,239 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:05,611 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:06,611 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:07,617 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:08,628 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:09,241 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:09,647 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:09,649 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:10,619 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:11,619 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:12,638 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:13,006 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:00:13,473 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:00:13,473 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:00:13,630 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:14,621 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:15,243 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:15,278 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:15,621 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:16,621 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:17,244 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:17,621 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:18,621 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:19,621 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:20,622 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:20,630 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:21,622 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:22,622 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:23,622 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:24,622 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:25,622 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:25,630 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:26,622 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:27,622 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:28,466 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:00:28,466 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:00:28,667 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:29,249 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:29,659 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:30,007 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 22:00:30,007 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_76fa4e81e9e73fef9cf0.mp4 with policy now +2025-01-15 22:00:30,032 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 22:00:30,032 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:00:30,033 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:00:30,033 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:00:30,249 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:00:30,250 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/15_eval_15_76fa4e81e9e73fef9cf0.mp4 +2025-01-15 22:00:30,250 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 22:00:30,263 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/858lh38v-media/videos/eval/15_eval_15_76fa4e81e9e73fef9cf0.mp4 +2025-01-15 22:00:30,537 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/pt3wspel-media/videos/eval/15_eval_15_76fa4e81e9e73fef9cf0.mp4 +2025-01-15 22:00:30,675 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:31,064 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:31,250 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:31,670 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:32,669 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:33,251 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:33,737 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:34,713 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:35,252 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:35,716 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:36,660 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:36,711 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:37,720 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:38,719 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:39,253 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:39,712 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:40,722 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:41,673 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:41,741 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:42,720 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:43,007 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:00:43,473 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:00:43,474 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:00:43,741 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:44,737 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:45,256 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:00:45,532 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/ia05v5to-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:00:45,555 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:00:45,555 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:00:45,555 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:00:45,744 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:46,256 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:00:46,732 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:46,953 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:47,257 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:47,734 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:48,744 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:49,747 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:50,777 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:51,258 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:51,768 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:52,187 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:52,748 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:53,767 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:54,769 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:55,781 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:56,783 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:57,670 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:00:57,802 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:58,454 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:00:58,455 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:00:58,759 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:00:59,262 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:00:59,759 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:00,759 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:01,263 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:01:01,761 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:02,760 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:03,648 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:03,760 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:04,761 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:05,761 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:06,761 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:07,761 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:08,648 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:08,761 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:09,762 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:10,761 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:11,762 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:12,762 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:13,007 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:01:13,454 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:01:13,454 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:01:13,648 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:13,762 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:14,762 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:15,763 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:16,763 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:17,763 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:18,649 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:18,763 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:19,763 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:20,764 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:21,764 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:22,764 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:23,649 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:23,764 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:24,764 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:25,813 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:26,274 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/16_eval_16_5019757f76965e10782b.mp4 +2025-01-15 22:01:26,274 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 22:01:26,287 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 22:01:26,288 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_5019757f76965e10782b.mp4 with policy now +2025-01-15 22:01:26,290 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 22:01:26,291 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:01:26,291 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:01:26,291 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:01:26,545 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/7disiecn-media/videos/eval/16_eval_16_5019757f76965e10782b.mp4 +2025-01-15 22:01:26,798 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:26,836 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/xc8psx8a-media/videos/eval/16_eval_16_5019757f76965e10782b.mp4 +2025-01-15 22:01:27,274 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:01:27,274 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:01:27,813 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:28,487 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:01:28,488 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:01:28,654 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:28,808 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:29,275 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:01:29,826 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:30,815 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:31,805 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:32,810 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:33,277 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:01:33,855 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:34,653 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:34,829 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:35,278 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:01:35,810 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:36,840 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:37,865 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:38,845 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:39,685 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:39,845 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:40,858 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:41,836 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:42,848 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:43,008 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:01:43,492 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:01:43,492 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:01:43,843 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:44,860 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:45,684 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:45,867 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:46,886 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:47,900 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:48,864 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:49,926 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:50,700 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:50,886 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:51,890 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:52,897 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:53,897 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:54,905 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:55,955 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:56,694 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:01:56,909 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:57,915 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:58,478 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:01:58,479 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:01:58,907 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:01:59,928 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:00,944 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:01,709 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:01,982 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:02,985 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:03,949 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:04,981 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:05,959 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:06,716 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:06,974 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:07,970 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:08,969 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:09,979 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:10,981 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:11,988 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:12,694 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:12,994 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:13,008 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:02:13,480 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:02:13,480 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:02:14,007 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:15,012 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:15,295 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:02:15,674 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:02:15,674 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:02:15,674 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:02:16,008 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:16,295 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:02:17,044 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:17,296 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:02:18,006 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:18,129 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:19,023 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:20,005 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:21,005 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:21,297 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:02:22,005 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:23,005 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:23,539 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:24,005 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:25,006 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:26,006 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:27,006 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:28,006 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:28,469 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:02:28,469 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:02:28,615 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:29,006 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:30,006 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:31,007 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:32,008 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:33,008 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:33,692 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:34,031 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:35,071 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:35,305 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:02:35,410 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 22:02:35,410 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_2f96b121fd6f99c60674.mp4 with policy now +2025-01-15 22:02:35,423 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 22:02:35,423 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:02:35,423 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:02:35,424 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:02:35,646 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/4yu7df0h-media/videos/eval/17_eval_17_2f96b121fd6f99c60674.mp4 +2025-01-15 22:02:36,063 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:36,306 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:02:36,306 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/17_eval_17_2f96b121fd6f99c60674.mp4 +2025-01-15 22:02:36,306 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 22:02:36,599 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/t1zmllnw-media/videos/eval/17_eval_17_2f96b121fd6f99c60674.mp4 +2025-01-15 22:02:37,044 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:37,306 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:02:38,080 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:38,731 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:39,059 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:40,076 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:41,061 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:41,308 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:02:42,069 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:43,009 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:02:43,089 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:43,507 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:02:43,508 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:02:44,099 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:44,730 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:45,113 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:45,310 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:02:46,115 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:47,119 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:48,115 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:49,135 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:49,738 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:50,127 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:51,153 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:52,146 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:53,151 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:54,157 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:54,741 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:02:55,171 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:56,148 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:57,153 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:58,197 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:58,315 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:02:58,482 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:02:58,483 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:02:58,725 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:02:58,725 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:02:58,725 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:02:59,185 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:02:59,316 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:02:59,316 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:02:59,316 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:03:00,173 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:00,401 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:01,164 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:01,317 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:02,188 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:03,191 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:04,190 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:05,173 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:05,318 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:05,734 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:06,203 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:07,205 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:08,201 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:09,207 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:10,194 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:11,134 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:11,194 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:11,321 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:12,194 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:13,009 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:03:13,194 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:13,478 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:03:13,478 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:03:14,194 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:15,195 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:16,195 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:16,789 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:17,195 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:18,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:19,195 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:20,195 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:21,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:22,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:22,723 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:23,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:24,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:25,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:26,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:27,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:27,723 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:28,196 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:28,478 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:03:28,479 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:03:29,197 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:30,197 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:31,197 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:32,197 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:32,723 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:33,197 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:34,197 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:35,197 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:36,197 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:37,212 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:37,725 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:38,199 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:38,518 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 22:03:38,518 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_c8c84019c8dc8ba0e4ea.mp4 with policy now +2025-01-15 22:03:38,526 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 22:03:38,526 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:03:38,526 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:03:38,527 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:03:38,777 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/n4px6l3n-media/videos/eval/18_eval_18_c8c84019c8dc8ba0e4ea.mp4 +2025-01-15 22:03:39,210 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:39,333 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:39,333 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:03:39,334 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/18_eval_18_c8c84019c8dc8ba0e4ea.mp4 +2025-01-15 22:03:39,334 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 22:03:39,649 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/szqdmjnl-media/videos/eval/18_eval_18_c8c84019c8dc8ba0e4ea.mp4 +2025-01-15 22:03:40,215 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:41,221 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:41,334 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:42,244 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:43,009 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:03:43,011 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:43,228 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:43,532 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:03:43,532 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:03:44,260 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:45,241 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:45,336 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:46,249 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:47,261 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:47,337 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:48,263 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:48,729 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:49,249 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:50,255 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:51,341 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:52,275 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:53,273 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:53,339 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:03:53,787 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:03:53,791 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:03:53,791 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:03:53,792 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:03:54,283 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:54,340 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:03:54,340 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:03:55,297 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:55,340 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:56,326 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:57,278 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:57,341 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:58,279 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:58,517 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:03:58,518 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:03:59,294 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:03:59,342 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:03:59,768 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:00,292 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:01,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:02,344 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:03,329 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:04,326 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:04,779 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:05,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:06,313 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:07,348 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:08,340 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:09,388 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:09,800 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:10,316 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:11,315 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:11,347 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:04:12,316 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:13,010 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:04:13,316 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:13,492 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:04:13,492 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:04:14,317 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:15,317 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:15,760 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:16,317 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:17,317 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:18,317 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:19,317 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:20,317 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:21,317 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:21,760 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:22,318 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:23,318 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:24,318 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:25,318 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:26,318 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:26,760 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:27,318 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:28,318 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:28,492 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:04:28,492 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:04:29,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:30,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:31,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:31,760 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:32,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:33,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:34,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:35,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:36,319 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:37,352 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:37,439 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:38,339 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 22:04:38,339 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_3f737e19fed74dd298b2.mp4 with policy now +2025-01-15 22:04:38,359 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/19_eval_19_3f737e19fed74dd298b2.mp4 +2025-01-15 22:04:38,359 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval +2025-01-15 22:04:38,370 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 22:04:38,377 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:04:38,377 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:38,377 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:38,379 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:38,606 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/tje53cgu-media/videos/eval/19_eval_19_3f737e19fed74dd298b2.mp4 +2025-01-15 22:04:38,908 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/s9jrxm23-media/videos/eval/19_eval_19_3f737e19fed74dd298b2.mp4 +2025-01-15 22:04:39,357 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:39,360 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:04:39,360 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:04:40,367 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:41,361 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:04:41,368 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:42,360 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:43,010 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:04:43,011 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:43,361 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:04:43,403 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:43,499 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:04:43,499 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:04:44,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:45,389 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:46,390 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:47,363 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:04:47,411 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:48,379 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:48,625 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:49,409 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:50,402 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:51,413 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:52,406 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:53,439 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:53,794 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:54,366 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:04:54,437 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:54,642 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/gms0en9u-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:04:54,864 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:04:54,864 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:54,864 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:55,367 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:04:55,367 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:04:55,367 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:04:55,446 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:55,970 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-15 22:04:55,971 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: log_artifact +2025-01-15 22:04:56,367 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table/eval/success_rate_ci_20_0f278608fd5017506fe4.table.json +2025-01-15 22:04:56,367 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media +2025-01-15 22:04:56,368 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table +2025-01-15 22:04:56,368 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table/eval +2025-01-15 22:04:56,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:56,449 INFO wandb-upload_3:2470873 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpnbdac6qz +2025-01-15 22:04:56,476 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 22:04:56,568 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/5fe6wu6w-media/table/eval/success_rate_ci_20_0f278608fd5017506fe4.table.json +2025-01-15 22:04:56,836 INFO SenderThread:2470873 [sender.py:send_request_log_artifact():1525] logged artifact run-roatk2bk-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0NDEwODI1NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzOTE1MDQ5NA==', 'latestArtifact': None}} +2025-01-15 22:04:56,836 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 22:04:56,836 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_0f278608fd5017506fe4.table.json with policy now +2025-01-15 22:04:56,836 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:04:56,836 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,837 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,837 DEBUG SenderThread:2470873 [sender.py:send():379] send: telemetry +2025-01-15 22:04:56,837 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,838 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,838 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,838 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,839 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,840 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,841 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,841 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,841 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,842 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,842 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,842 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,842 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,843 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,843 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,843 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,844 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,844 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,844 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,845 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,845 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:56,845 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:56,875 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-15 22:04:56,875 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: log_artifact +2025-01-15 22:04:56,887 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 22:04:57,140 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/ve85kqna-media/table/eval/success_rate_ci_20_0f278608fd5017506fe4.table.json +2025-01-15 22:04:57,368 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:04:57,368 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:04:57,368 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table/eval/total_success_rate_ci_21_ee38b3327b46c4981cc4.table.json +2025-01-15 22:04:57,368 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table/eval +2025-01-15 22:04:57,389 INFO wandb-upload_3:2470873 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpuhxzzwc7 +2025-01-15 22:04:57,406 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:57,563 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/44kyffge-media/table/eval/total_success_rate_ci_21_ee38b3327b46c4981cc4.table.json +2025-01-15 22:04:57,762 INFO SenderThread:2470873 [sender.py:send_request_log_artifact():1525] logged artifact run-roatk2bk-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0NDEwODI2Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzOTE1MDQ5Ng==', 'latestArtifact': None}} +2025-01-15 22:04:57,762 DEBUG SenderThread:2470873 [sender.py:send():379] send: files +2025-01-15 22:04:57,762 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_ee38b3327b46c4981cc4.table.json with policy now +2025-01-15 22:04:57,763 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:04:57,763 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:04:57,763 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:04:58,031 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/ydokvtt1-media/table/eval/total_success_rate_ci_21_ee38b3327b46c4981cc4.table.json +2025-01-15 22:04:58,368 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:04:58,406 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:04:58,496 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:04:58,496 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:04:58,798 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:04:59,369 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:04:59,406 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:00,406 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:01,406 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:02,406 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:03,406 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:03,802 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:05:04,372 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/config.yaml +2025-01-15 22:05:04,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:05,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:06,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:07,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:08,407 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:09,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:09,799 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:05:10,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:11,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:12,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:13,011 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:05:13,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:13,496 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 22:05:13,496 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: stop_status +2025-01-15 22:05:14,408 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:14,841 DEBUG SenderThread:2470873 [sender.py:send():379] send: exit +2025-01-15 22:05:14,841 INFO SenderThread:2470873 [sender.py:send_exit():586] handling exit code: 0 +2025-01-15 22:05:14,842 INFO SenderThread:2470873 [sender.py:send_exit():588] handling runtime: 1201 +2025-01-15 22:05:14,842 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:05:14,842 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:05:14,842 INFO SenderThread:2470873 [sender.py:send_exit():594] send defer +2025-01-15 22:05:14,842 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:14,842 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-15 22:05:14,843 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:14,843 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-15 22:05:14,843 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 1 +2025-01-15 22:05:14,843 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:14,843 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-15 22:05:14,843 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:14,843 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-15 22:05:14,843 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 2 +2025-01-15 22:05:14,843 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:14,843 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-15 22:05:14,843 INFO HandlerThread:2470873 [system_monitor.py:finish():203] Stopping system monitor +2025-01-15 22:05:14,843 DEBUG SystemMonitor:2470873 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-15 22:05:14,844 INFO HandlerThread:2470873 [interfaces.py:finish():200] Joined cpu monitor +2025-01-15 22:05:14,844 DEBUG SystemMonitor:2470873 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-15 22:05:14,844 INFO HandlerThread:2470873 [interfaces.py:finish():200] Joined disk monitor +2025-01-15 22:05:14,915 INFO HandlerThread:2470873 [interfaces.py:finish():200] Joined gpu monitor +2025-01-15 22:05:14,915 INFO HandlerThread:2470873 [interfaces.py:finish():200] Joined memory monitor +2025-01-15 22:05:14,915 INFO HandlerThread:2470873 [interfaces.py:finish():200] Joined network monitor +2025-01-15 22:05:14,915 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:14,915 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-15 22:05:14,916 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 3 +2025-01-15 22:05:14,916 DEBUG SenderThread:2470873 [sender.py:send():379] send: stats +2025-01-15 22:05:14,916 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:14,916 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-15 22:05:14,916 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:14,916 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-15 22:05:14,916 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 4 +2025-01-15 22:05:14,916 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:14,916 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-15 22:05:15,378 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:05:17,379 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:05:21,872 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 22:05:21,873 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:05:21,873 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: keepalive +2025-01-15 22:05:21,874 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:21,874 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-15 22:05:21,874 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 5 +2025-01-15 22:05:21,874 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 22:05:21,874 DEBUG SenderThread:2470873 [sender.py:send():379] send: history +2025-01-15 22:05:21,874 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: summary_record +2025-01-15 22:05:21,874 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:21,875 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-15 22:05:21,875 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:05:21,875 DEBUG SenderThread:2470873 [sender.py:send():379] send: summary +2025-01-15 22:05:21,879 INFO SenderThread:2470873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 22:05:21,879 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:21,879 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-15 22:05:21,879 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 6 +2025-01-15 22:05:21,879 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:21,880 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-15 22:05:21,880 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:21,880 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-15 22:05:21,880 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 7 +2025-01-15 22:05:21,880 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 22:05:21,880 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:21,880 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-15 22:05:21,880 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:21,880 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-15 22:05:22,381 INFO Thread-12 :2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:05:22,843 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 22:05:23,031 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 8 +2025-01-15 22:05:23,031 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 22:05:23,031 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:23,031 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-15 22:05:23,031 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:23,032 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-15 22:05:23,032 INFO SenderThread:2470873 [job_builder.py:build():440] Attempting to build job artifact +2025-01-15 22:05:23,035 INFO SenderThread:2470873 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-15 22:05:23,036 INFO SenderThread:2470873 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-15 22:05:23,049 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 9 +2025-01-15 22:05:23,049 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:23,049 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-15 22:05:23,050 DEBUG SenderThread:2470873 [sender.py:send():379] send: artifact +2025-01-15 22:05:23,221 INFO SenderThread:2470873 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ0NDA3MjMyOQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ0NDA3MjMyOQ==', 'versionIndex': 440}}} +2025-01-15 22:05:23,221 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:23,221 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-15 22:05:23,221 INFO SenderThread:2470873 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-15 22:05:23,382 INFO SenderThread:2470873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:05:23,383 INFO SenderThread:2470873 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files +2025-01-15 22:05:23,383 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/config.yaml config.yaml +2025-01-15 22:05:23,383 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json wandb-summary.json +2025-01-15 22:05:23,383 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/requirements.txt requirements.txt +2025-01-15 22:05:23,386 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-metadata.json wandb-metadata.json +2025-01-15 22:05:23,386 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:05:23,386 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-15 22:05:23,386 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log output.log +2025-01-15 22:05:23,388 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/diff.patch diff.patch +2025-01-15 22:05:23,388 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-15 22:05:23,388 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/3_eval_3_89821fb09b372630838b.mp4 media/videos/eval/3_eval_3_89821fb09b372630838b.mp4 +2025-01-15 22:05:23,388 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/4_eval_4_9fb52356d0a213033cfe.mp4 media/videos/eval/4_eval_4_9fb52356d0a213033cfe.mp4 +2025-01-15 22:05:23,388 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/10_eval_10_6e43fe2801f3a905e582.mp4 media/videos/eval/10_eval_10_6e43fe2801f3a905e582.mp4 +2025-01-15 22:05:23,388 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/5_eval_5_a69fcdfd49e2066116b6.mp4 media/videos/eval/5_eval_5_a69fcdfd49e2066116b6.mp4 +2025-01-15 22:05:23,388 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/0_eval_0_2eb6cf19687a4fabfe3c.mp4 media/videos/eval/0_eval_0_2eb6cf19687a4fabfe3c.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/18_eval_18_c8c84019c8dc8ba0e4ea.mp4 media/videos/eval/18_eval_18_c8c84019c8dc8ba0e4ea.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/15_eval_15_76fa4e81e9e73fef9cf0.mp4 media/videos/eval/15_eval_15_76fa4e81e9e73fef9cf0.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/19_eval_19_3f737e19fed74dd298b2.mp4 media/videos/eval/19_eval_19_3f737e19fed74dd298b2.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/17_eval_17_2f96b121fd6f99c60674.mp4 media/videos/eval/17_eval_17_2f96b121fd6f99c60674.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/12_eval_12_241be90b3f1c54884e79.mp4 media/videos/eval/12_eval_12_241be90b3f1c54884e79.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/13_eval_13_cb3449f02cfc7cf55b8d.mp4 media/videos/eval/13_eval_13_cb3449f02cfc7cf55b8d.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/1_eval_1_b5d7870b7b04a6ef485f.mp4 media/videos/eval/1_eval_1_b5d7870b7b04a6ef485f.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/2_eval_2_f98c31cd2e05324cffd8.mp4 media/videos/eval/2_eval_2_f98c31cd2e05324cffd8.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/6_eval_6_ae418a6c99a51b4201d6.mp4 media/videos/eval/6_eval_6_ae418a6c99a51b4201d6.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/14_eval_14_074bdfc80a4a9d765f98.mp4 media/videos/eval/14_eval_14_074bdfc80a4a9d765f98.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/16_eval_16_5019757f76965e10782b.mp4 media/videos/eval/16_eval_16_5019757f76965e10782b.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/7_eval_7_f02cea2dbceefebb3b97.mp4 media/videos/eval/7_eval_7_f02cea2dbceefebb3b97.mp4 +2025-01-15 22:05:23,389 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/9_eval_9_19795df8831a10c10837.mp4 media/videos/eval/9_eval_9_19795df8831a10c10837.mp4 +2025-01-15 22:05:23,390 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/8_eval_8_fb8f14c1c353cc80983a.mp4 media/videos/eval/8_eval_8_fb8f14c1c353cc80983a.mp4 +2025-01-15 22:05:23,390 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/videos/eval/11_eval_11_4263c7dc48b07a1c552c.mp4 media/videos/eval/11_eval_11_4263c7dc48b07a1c552c.mp4 +2025-01-15 22:05:23,390 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table/eval/total_success_rate_ci_21_ee38b3327b46c4981cc4.table.json media/table/eval/total_success_rate_ci_21_ee38b3327b46c4981cc4.table.json +2025-01-15 22:05:23,390 INFO SenderThread:2470873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/media/table/eval/success_rate_ci_20_0f278608fd5017506fe4.table.json media/table/eval/success_rate_ci_20_0f278608fd5017506fe4.table.json +2025-01-15 22:05:23,390 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 10 +2025-01-15 22:05:23,392 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:23,393 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-15 22:05:23,393 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:23,393 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-15 22:05:23,393 INFO SenderThread:2470873 [file_pusher.py:finish():169] shutting down file pusher +2025-01-15 22:05:23,569 INFO wandb-upload_2:2470873 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/wandb-summary.json +2025-01-15 22:05:23,575 INFO wandb-upload_3:2470873 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/config.yaml +2025-01-15 22:05:23,597 INFO wandb-upload_1:2470873 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/requirements.txt +2025-01-15 22:05:23,660 INFO wandb-upload_4:2470873 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/files/output.log +2025-01-15 22:05:23,707 INFO wandb-upload_0:2470873 [upload_job.py:push():130] Uploaded file /tmp/tmpt0febobwwandb/niuudu4q-events.out.tfevents.1736995513.leonmkim-ROG-Strix-G15CS-G15CS.2470707.0 +2025-01-15 22:05:23,843 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 22:05:23,843 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 22:05:23,908 INFO Thread-11 (_thread_body):2470873 [sender.py:transition_state():614] send defer: 11 +2025-01-15 22:05:23,908 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:23,908 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-15 22:05:23,908 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:23,908 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-15 22:05:23,908 INFO SenderThread:2470873 [file_pusher.py:join():175] waiting for file pusher +2025-01-15 22:05:23,916 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 12 +2025-01-15 22:05:23,916 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:23,916 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-15 22:05:23,916 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:23,916 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-15 22:05:23,916 INFO SenderThread:2470873 [file_stream.py:finish():601] file stream finish called +2025-01-15 22:05:24,132 INFO SenderThread:2470873 [file_stream.py:finish():605] file stream finish is done +2025-01-15 22:05:24,132 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 13 +2025-01-15 22:05:24,133 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:24,133 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-15 22:05:24,133 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:24,133 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-15 22:05:24,133 INFO SenderThread:2470873 [sender.py:transition_state():614] send defer: 14 +2025-01-15 22:05:24,133 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: defer +2025-01-15 22:05:24,133 INFO HandlerThread:2470873 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-15 22:05:24,133 DEBUG SenderThread:2470873 [sender.py:send():379] send: final +2025-01-15 22:05:24,133 DEBUG SenderThread:2470873 [sender.py:send():379] send: footer +2025-01-15 22:05:24,133 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: defer +2025-01-15 22:05:24,133 INFO SenderThread:2470873 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-15 22:05:24,134 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 22:05:24,134 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: server_info +2025-01-15 22:05:24,134 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 22:05:24,134 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 22:05:24,135 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: server_info +2025-01-15 22:05:24,136 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: get_summary +2025-01-15 22:05:24,137 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-15 22:05:24,138 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 22:05:24,187 DEBUG SenderThread:2470873 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 22:05:24,188 INFO MainThread:2470873 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-15 22:05:24,188 INFO MainThread:2470873 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-15 22:05:24,189 INFO MainThread:2470873 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-15 22:05:24,189 DEBUG HandlerThread:2470873 [handler.py:handle_request():158] handle_request: shutdown +2025-01-15 22:05:24,189 INFO HandlerThread:2470873 [handler.py:finish():882] shutting down handler +2025-01-15 22:05:25,135 INFO WriterThread:2470873 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/run-roatk2bk.wandb +2025-01-15 22:05:25,188 INFO SenderThread:2470873 [sender.py:finish():1615] shutting down sender +2025-01-15 22:05:25,188 INFO SenderThread:2470873 [file_pusher.py:finish():169] shutting down file pusher +2025-01-15 22:05:25,188 INFO SenderThread:2470873 [file_pusher.py:join():175] waiting for file pusher diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/logs/debug.log b/roatk2bk/wandb/run-20250115_214512-roatk2bk/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..90333c4a07365b51ebf0a693ef911a3a0ac455c9 --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-15 21:45:12,618 INFO MainThread:2470707 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-15 21:45:12,618 INFO MainThread:2470707 [wandb_setup.py:_flush():76] Configure stats pid to 2470707 +2025-01-15 21:45:12,618 INFO MainThread:2470707 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/settings +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/logs/debug.log +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/wandb/run-20250115_214512-roatk2bk/logs/debug-internal.log +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_init.py:init():569] calling init triggers +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3456_0', 'wandb_notes': '3456_0_req_3921_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk'} +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_init.py:init():619] starting backend +2025-01-15 21:45:12,619 INFO MainThread:2470707 [wandb_init.py:init():623] setting up manager +2025-01-15 21:45:12,623 INFO MainThread:2470707 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-15 21:45:12,624 INFO MainThread:2470707 [wandb_init.py:init():631] backend started and connected +2025-01-15 21:45:12,637 INFO MainThread:2470707 [wandb_init.py:init():720] updated telemetry +2025-01-15 21:45:12,642 INFO MainThread:2470707 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-15 21:45:12,911 INFO MainThread:2470707 [wandb_run.py:_on_init():2435] communicating current version +2025-01-15 21:45:12,981 INFO MainThread:2470707 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.3 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-15 21:45:12,981 INFO MainThread:2470707 [wandb_init.py:init():804] starting run threads in backend +2025-01-15 21:45:13,359 INFO MainThread:2470707 [wandb_run.py:_console_start():2413] atexit reg +2025-01-15 21:45:13,359 INFO MainThread:2470707 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-15 21:45:13,359 INFO MainThread:2470707 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-15 21:45:13,359 INFO MainThread:2470707 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-15 21:45:13,363 INFO MainThread:2470707 [wandb_init.py:init():847] run started, returning control to user process +2025-01-15 21:45:13,363 INFO MainThread:2470707 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/roatk2bk/tb, True +2025-01-15 21:45:41,677 INFO MainThread:2470707 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-15 22:05:25,189 WARNING MsgRouterThr:2470707 [router.py:message_loop():77] message_loop has been closed diff --git a/roatk2bk/wandb/run-20250115_214512-roatk2bk/run-roatk2bk.wandb b/roatk2bk/wandb/run-20250115_214512-roatk2bk/run-roatk2bk.wandb new file mode 100644 index 0000000000000000000000000000000000000000..acd9eef65c3ec9488d69d5a9c5206f194ff6741d --- /dev/null +++ b/roatk2bk/wandb/run-20250115_214512-roatk2bk/run-roatk2bk.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f99913c395489ce012f2ed1f8492ffe3e3c21a1930cdcfb6b885c762b98ad4d3 +size 247948