diff --git a/.gitattributes b/.gitattributes index 322e2543c47fb2c7c6f0326ecfebf8aaa473c467..e00fbdf22597c07cfb030bed78fe82d5235e539c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -273,3 +273,49 @@ mo0b6mh9/episode_rosbags/episode_10_2025-01-18-20-03-04.bag filter=lfs diff=lfs h3eacbh9/episode_rosbags/episode_5_2025-01-18-20-18-13.bag filter=lfs diff=lfs merge=lfs -text bn7ul4i2/episode_rosbags/episode_4_2025-01-21-21-10-05.bag filter=lfs diff=lfs merge=lfs -text q1ti958r/episode_rosbags/episode_14_2025-01-18-19-00-54.bag filter=lfs diff=lfs merge=lfs -text +h3eacbh9/episode_rosbags/episode_6_2025-01-18-20-18-58.bag filter=lfs diff=lfs merge=lfs -text +q1ti958r/episode_rosbags/episode_17_2025-01-18-19-04-12.bag filter=lfs diff=lfs merge=lfs -text +q1ti958r/episode_rosbags/episode_1_2025-01-18-18-51-19.bag filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/run-h3eacbh9.wandb filter=lfs diff=lfs merge=lfs -text +h3eacbh9/episode_rosbags/episode_15_2025-01-18-20-25-57.bag filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/3_eval_3_c4ac9168429d53521dfd.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/6_eval_6_6f92e10ddc5700f95864.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/9_eval_9_45f46258144727274eca.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/18_eval_18_666566ff2e12fdecef30.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/episode_rosbags/episode_3_2025-01-18-20-16-46.bag filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/5_eval_5_8eee7b4209c8cc001b29.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/7_eval_7_ec916f9f1cb86bb268d4.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/15_eval_15_a134ab49305f64bfef91.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/1_eval_1_c93c245f94bccf988bf8.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/17_eval_17_70585fb0ca24437a0f9f.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/2_eval_2_35d91a735aea17ce0770.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/11_eval_11_359c87ada34d682bae45.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/10_eval_10_dbfaa8aaf0c838051064.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/4_eval_4_73cfbcf4697dd02a1704.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/14_eval_14_d43b1e73114608f44587.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/13_eval_13_114ad2ddaaeca4cacdd5.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/8_eval_8_c1b631f94f507d0b8029.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/12_eval_12_50069bff80de5f046230.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/0_eval_0_958873708057cb5c96d0.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/16_eval_16_fba70f3485b0b97a5e73.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/19_eval_19_d8301c722455e230f538.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/19_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/7_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/12_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/8_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/18_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/6_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/9_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/5_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/15_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/11_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/16_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/17_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/13_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/14_eval.mp4 filter=lfs diff=lfs merge=lfs -text +h3eacbh9/eval_video/10_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/h3eacbh9/episode_rosbags/color_K.npy b/h3eacbh9/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/h3eacbh9/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/h3eacbh9/episode_rosbags/episode_15_2025-01-18-20-25-57.bag b/h3eacbh9/episode_rosbags/episode_15_2025-01-18-20-25-57.bag new file mode 100644 index 0000000000000000000000000000000000000000..4f9a6125a49e9d0955dba7490e62cd94c42eeeb9 --- /dev/null +++ b/h3eacbh9/episode_rosbags/episode_15_2025-01-18-20-25-57.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:587d7063581d7ebb218f538c1316e2b8898f5f36d398193d3ceca49793981448 +size 1410806768 diff --git a/h3eacbh9/episode_rosbags/episode_3_2025-01-18-20-16-46.bag b/h3eacbh9/episode_rosbags/episode_3_2025-01-18-20-16-46.bag new file mode 100644 index 0000000000000000000000000000000000000000..1829366acaba39888ecc14a067a4ebf4927a9384 --- /dev/null +++ b/h3eacbh9/episode_rosbags/episode_3_2025-01-18-20-16-46.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:576f21622ad6423805c4a8200602ee779d10f45fe4a3caaee6e8d5ef0efdfdcd +size 1409388038 diff --git a/h3eacbh9/episode_rosbags/episode_6_2025-01-18-20-18-58.bag b/h3eacbh9/episode_rosbags/episode_6_2025-01-18-20-18-58.bag new file mode 100644 index 0000000000000000000000000000000000000000..3080100a725d7cf2ef1d68bd5ca7da72b430f21b --- /dev/null +++ b/h3eacbh9/episode_rosbags/episode_6_2025-01-18-20-18-58.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2819a02e187e7a3cb1118dd6b58823987e596e3ecbdf5deae333357c514e37f0 +size 1410031567 diff --git a/h3eacbh9/eval_video/0_eval.mp4 b/h3eacbh9/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5da2dbdf8af84a1e2ae7e310e94e2a79fdf56309 --- /dev/null +++ b/h3eacbh9/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:958873708057cb5c96d07eec36f6c0e68759bdf4c854f4fae4331f95dc510299 +size 1142998 diff --git a/h3eacbh9/eval_video/10_eval.mp4 b/h3eacbh9/eval_video/10_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bce106247fd8700eeb72e3df233bd6153c9dffba --- /dev/null +++ b/h3eacbh9/eval_video/10_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbfaa8aaf0c838051064d1f52334d6d83d61935c556bc18851fb329b8e51a97b +size 1290176 diff --git a/h3eacbh9/eval_video/11_eval.mp4 b/h3eacbh9/eval_video/11_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..da117112871f053c9eca5be46c5831b36dc9b5ee --- /dev/null +++ b/h3eacbh9/eval_video/11_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:359c87ada34d682bae4526c67a0e1660ebc14229bbfea9ebda9878bc7cbe49e2 +size 1165825 diff --git a/h3eacbh9/eval_video/12_eval.mp4 b/h3eacbh9/eval_video/12_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..80264a1be329e3d40f7a32794abce6c751fbcf8e --- /dev/null +++ b/h3eacbh9/eval_video/12_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50069bff80de5f04623040da25c9abba38c19c8c6f485f7ecb364ef9c652a551 +size 1365675 diff --git a/h3eacbh9/eval_video/13_eval.mp4 b/h3eacbh9/eval_video/13_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..380b896adbd40c221465988facaa77cfede9743b --- /dev/null +++ b/h3eacbh9/eval_video/13_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:114ad2ddaaeca4cacdd5ae91c8134dd40a3d5b8d16f5c1538175afff73d03d34 +size 1346252 diff --git a/h3eacbh9/eval_video/14_eval.mp4 b/h3eacbh9/eval_video/14_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..494ea1cd16adaf7756f4448240608d5e53f89ccb --- /dev/null +++ b/h3eacbh9/eval_video/14_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d43b1e73114608f44587bfbbd894e999f6f34aa1b616cbab8d4f16ce3f271adc +size 1413320 diff --git a/h3eacbh9/eval_video/15_eval.mp4 b/h3eacbh9/eval_video/15_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9c219d50216d5ecaed401e2afe56d20990d9355e --- /dev/null +++ b/h3eacbh9/eval_video/15_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a134ab49305f64bfef917aadcc5002acbf1b6c847787e9add516af98661440cb +size 1461649 diff --git a/h3eacbh9/eval_video/16_eval.mp4 b/h3eacbh9/eval_video/16_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..11fa3d1f2ebe734751ca63f5468d8fbfa80c246c --- /dev/null +++ b/h3eacbh9/eval_video/16_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fba70f3485b0b97a5e738e162bd29deed4662a79fc5279012753333773ee3b10 +size 1472547 diff --git a/h3eacbh9/eval_video/17_eval.mp4 b/h3eacbh9/eval_video/17_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..287815734eed0c3edcf3abe89a3d31753632fec9 --- /dev/null +++ b/h3eacbh9/eval_video/17_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70585fb0ca24437a0f9f8d8ce89db96ec7d90d0e2abf545ad2ba5f0a1b9d31c9 +size 1504233 diff --git a/h3eacbh9/eval_video/18_eval.mp4 b/h3eacbh9/eval_video/18_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..03ba6751ae466d8cacf2a9d82309f2e1678fdbbd --- /dev/null +++ b/h3eacbh9/eval_video/18_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:666566ff2e12fdecef308d665951ca56d112b39565b2350089304d70dea6847b +size 1399960 diff --git a/h3eacbh9/eval_video/19_eval.mp4 b/h3eacbh9/eval_video/19_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e15b7c1d3d70e24137016d81d0c19d1556e4d6f5 --- /dev/null +++ b/h3eacbh9/eval_video/19_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8301c722455e230f538b18357aed48ab0500a9465cfbf6fbf0edd142fcf14c5 +size 1567147 diff --git a/h3eacbh9/eval_video/1_eval.mp4 b/h3eacbh9/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..60db094cdc32c9ad320f819b51f1ec791a148f7f --- /dev/null +++ b/h3eacbh9/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c93c245f94bccf988bf81c13b34f2b4f94a31c61b0b700a37cd4e15dacb444df +size 1151340 diff --git a/h3eacbh9/eval_video/2_eval.mp4 b/h3eacbh9/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b5cf0756fad4a918e8263bc2b1c22d1082f5d0f8 --- /dev/null +++ b/h3eacbh9/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35d91a735aea17ce0770b204540d9606ca19ba47596be1fa84c03e8a76d48b12 +size 1160864 diff --git a/h3eacbh9/eval_video/3_eval.mp4 b/h3eacbh9/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a923a768bccd7cc67a2904cac6051f895498872a --- /dev/null +++ b/h3eacbh9/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4ac9168429d53521dfd398dfdf30167e676a6d017c37e6719de8c077ae2d6c5 +size 1157641 diff --git a/h3eacbh9/eval_video/4_eval.mp4 b/h3eacbh9/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5de845cb75f77a1083eb11c7bd1ac2d5c69f18b4 --- /dev/null +++ b/h3eacbh9/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73cfbcf4697dd02a170455072cbbb3a1ca0352372cb4a0797e92ead11895af34 +size 1154605 diff --git a/h3eacbh9/eval_video/5_eval.mp4 b/h3eacbh9/eval_video/5_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7ed8e7be99a9929e18fa2a68d5a9d82a13af8feb --- /dev/null +++ b/h3eacbh9/eval_video/5_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eee7b4209c8cc001b291cef3b235d9ad935ca7d79ef91da31d83f4b2565d510 +size 1146010 diff --git a/h3eacbh9/eval_video/6_eval.mp4 b/h3eacbh9/eval_video/6_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d322d029bc8363ed25b74e667d4a2583292a3d2f --- /dev/null +++ b/h3eacbh9/eval_video/6_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f92e10ddc5700f95864b3bd37bb543a48502145e8956f152881dac10ecc3e27 +size 1142252 diff --git a/h3eacbh9/eval_video/7_eval.mp4 b/h3eacbh9/eval_video/7_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7a131da3e5da8113f973d0ca4ac62ed5df185e93 --- /dev/null +++ b/h3eacbh9/eval_video/7_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec916f9f1cb86bb268d4b76b9be7a6f2b0e4f3512ce092e5d968e616457e41ba +size 1230184 diff --git a/h3eacbh9/eval_video/8_eval.mp4 b/h3eacbh9/eval_video/8_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5a776ce207ea07d608e72c7c35ee5a73f965ecb8 --- /dev/null +++ b/h3eacbh9/eval_video/8_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1b631f94f507d0b802907ef582d02c763dd6faa3b1b3b36bfa5e09e9ab836bc +size 1261905 diff --git a/h3eacbh9/eval_video/9_eval.mp4 b/h3eacbh9/eval_video/9_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..10236d1d5824622ddd2869d703ca5ddaade6c832 --- /dev/null +++ b/h3eacbh9/eval_video/9_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45f46258144727274eca27c01c0f84d776e772cc618e870d73101f7b953894fb +size 1162867 diff --git a/h3eacbh9/tb/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 b/h3eacbh9/tb/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 new file mode 100644 index 0000000000000000000000000000000000000000..0563dcfdda83443d525d3248bfec82c93ec272de --- /dev/null +++ b/h3eacbh9/tb/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a3e001b5005b838120627dd736775409f4f91112a2e56381e30b3e2c1ab3212 +size 22784 diff --git a/h3eacbh9/wandb/debug-internal.log b/h3eacbh9/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d938868f7208fce03fe29a0b209acee1e9f1ed1b --- /dev/null +++ b/h3eacbh9/wandb/debug-internal.log @@ -0,0 +1,2207 @@ +2025-01-18 20:14:03,266 INFO StreamThr :2569743 [internal.py:wandb_internal():85] W&B internal server running at pid: 2569743, started at: 2025-01-18 20:14:03.266253 +2025-01-18 20:14:03,267 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status +2025-01-18 20:14:03,279 INFO WriterThread:2569743 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/run-h3eacbh9.wandb +2025-01-18 20:14:03,279 DEBUG SenderThread:2569743 [sender.py:send():379] send: header +2025-01-18 20:14:03,286 DEBUG SenderThread:2569743 [sender.py:send():379] send: run +2025-01-18 20:14:03,527 INFO SenderThread:2569743 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files +2025-01-18 20:14:03,527 INFO SenderThread:2569743 [sender.py:_start_run_threads():1188] run started: h3eacbh9 with start time 1737249243.26689 +2025-01-18 20:14:03,542 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: check_version +2025-01-18 20:14:03,542 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: check_version +2025-01-18 20:14:03,621 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: run_start +2025-01-18 20:14:03,633 DEBUG HandlerThread:2569743 [system_info.py:__init__():26] System info init +2025-01-18 20:14:03,633 DEBUG HandlerThread:2569743 [system_info.py:__init__():41] System info init done +2025-01-18 20:14:03,633 INFO HandlerThread:2569743 [system_monitor.py:start():194] Starting system monitor +2025-01-18 20:14:03,633 INFO SystemMonitor:2569743 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-18 20:14:03,633 INFO HandlerThread:2569743 [system_monitor.py:probe():214] Collecting system info +2025-01-18 20:14:03,634 INFO SystemMonitor:2569743 [interfaces.py:start():188] Started cpu monitoring +2025-01-18 20:14:03,635 INFO SystemMonitor:2569743 [interfaces.py:start():188] Started disk monitoring +2025-01-18 20:14:03,635 INFO SystemMonitor:2569743 [interfaces.py:start():188] Started gpu monitoring +2025-01-18 20:14:03,636 INFO SystemMonitor:2569743 [interfaces.py:start():188] Started memory monitoring +2025-01-18 20:14:03,636 INFO SystemMonitor:2569743 [interfaces.py:start():188] Started network monitoring +2025-01-18 20:14:03,646 DEBUG HandlerThread:2569743 [system_info.py:probe():152] Probing system +2025-01-18 20:14:03,648 DEBUG HandlerThread:2569743 [system_info.py:_probe_git():137] Probing git +2025-01-18 20:14:03,652 DEBUG HandlerThread:2569743 [system_info.py:_probe_git():145] Probing git done +2025-01-18 20:14:03,652 DEBUG HandlerThread:2569743 [system_info.py:probe():200] Probing system done +2025-01-18 20:14:03,652 DEBUG HandlerThread:2569743 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-19T01:14:03.646859', 'startedAt': '2025-01-19T01:14:03.251322', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.5105, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4492.066, 'min': 800.0, 'max': 4700.0}, {'current': 4500.032, 'min': 800.0, 'max': 4700.0}, {'current': 4500.021, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 605.3610420227051}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-18 20:14:03,653 INFO HandlerThread:2569743 [system_monitor.py:probe():224] Finished collecting system info +2025-01-18 20:14:03,653 INFO HandlerThread:2569743 [system_monitor.py:probe():227] Publishing system info +2025-01-18 20:14:03,653 DEBUG HandlerThread:2569743 [system_info.py:_save_code():44] Saving code +2025-01-18 20:14:03,658 DEBUG HandlerThread:2569743 [system_info.py:_save_code():67] Saving code done +2025-01-18 20:14:03,658 DEBUG HandlerThread:2569743 [system_info.py:_save_patches():84] Saving git patches +2025-01-18 20:14:03,753 DEBUG HandlerThread:2569743 [system_info.py:_save_patches():126] Saving git patches done +2025-01-18 20:14:03,753 INFO HandlerThread:2569743 [system_monitor.py:probe():229] Finished publishing system info +2025-01-18 20:14:03,757 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:14:03,757 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-18 20:14:03,757 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-18 20:14:03,757 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-18 20:14:03,757 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-18 20:14:03,984 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: python_packages +2025-01-18 20:14:03,985 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: python_packages +2025-01-18 20:14:03,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:14:03,985 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:14:03,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:03,987 INFO HandlerThread:2569743 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/tb" + save: true +} +control { + relay_id: "177c0a6323c5493290332c4877990993" +} +_info { + stream_id: "h3eacbh9" +} + +2025-01-18 20:14:03,988 DEBUG HandlerThread:2569743 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-18 20:14:04,032 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/y2s8i8aq-wandb-metadata.json +2025-01-18 20:14:04,035 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/e8ijb1ii-code/FISH/eval_robot.py +2025-01-18 20:14:04,052 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/wdl6mw9j-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 20:14:04,053 DEBUG SenderThread:2569743 [sender.py:send():379] send: telemetry +2025-01-18 20:14:04,054 DEBUG SenderThread:2569743 [sender.py:send():379] send: tbrecord +2025-01-18 20:14:04,054 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:14:04,054 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 with policy live +2025-01-18 20:14:04,060 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/8mksntvi-diff.patch +2025-01-18 20:14:04,246 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/yyc7lpp2-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:14:04,531 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/requirements.txt +2025-01-18 20:14:04,531 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 20:14:04,531 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/diff.patch +2025-01-18 20:14:04,532 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:14:04,532 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-metadata.json +2025-01-18 20:14:04,532 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/code/FISH/eval_robot.py +2025-01-18 20:14:04,533 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:04,533 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/code/FISH +2025-01-18 20:14:04,533 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/code +2025-01-18 20:14:04,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:05,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:06,528 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:06,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:07,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:08,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:08,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:09,078 DEBUG SenderThread:2569743 [sender.py:send():379] send: config +2025-01-18 20:14:09,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:10,529 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:10,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:11,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:12,530 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:12,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:13,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:13,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:14,531 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:14,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:15,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:16,531 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:16,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:17,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:18,532 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:18,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:14:18,985 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:14:19,029 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:19,048 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:19,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:20,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:21,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:22,533 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:22,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:23,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:24,155 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:24,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:25,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:26,534 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:26,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:27,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:28,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:29,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:29,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:30,989 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:31,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:32,989 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:33,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:14:33,985 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:14:34,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:34,989 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:35,067 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:35,537 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/config.yaml +2025-01-18 20:14:35,989 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:36,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:37,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:38,989 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:39,989 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:40,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:40,989 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:41,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:42,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:43,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:44,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:45,991 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:46,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:46,991 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:47,991 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:48,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:14:48,985 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:14:48,991 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:49,991 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:50,540 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:50,991 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:51,916 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:14:51,917 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:14:51,917 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_958873708057cb5c96d0.mp4 with policy now +2025-01-18 20:14:51,918 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:14:51,918 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:14:51,918 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:14:51,991 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:52,192 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/7e8whsay-media/videos/eval/0_eval_0_958873708057cb5c96d0.mp4 +2025-01-18 20:14:52,541 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:52,541 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:14:52,542 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/0_eval_0_958873708057cb5c96d0.mp4 +2025-01-18 20:14:52,542 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:14:52,542 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos +2025-01-18 20:14:52,542 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media +2025-01-18 20:14:52,848 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/sufv42hu-media/videos/eval/0_eval_0_958873708057cb5c96d0.mp4 +2025-01-18 20:14:52,924 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:52,992 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:53,992 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:54,542 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:54,992 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:55,992 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:56,543 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:56,992 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:57,993 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:58,236 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:58,993 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:59,993 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:00,544 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:00,993 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:01,993 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:02,993 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:03,636 DEBUG SystemMonitor:2569743 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-18 20:15:03,637 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:15:03,637 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:03,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:15:03,986 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:15:04,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:04,993 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:05,993 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:06,993 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:07,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:08,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:08,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:09,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:10,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:11,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:12,548 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:15:12,806 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/3xm431t2-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:15:12,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:13,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:13,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:14,549 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:14,995 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:15,995 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:16,995 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:17,995 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:18,550 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:18,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:15:18,986 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:15:19,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:19,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:19,995 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:20,995 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:21,996 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:22,996 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:23,996 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:24,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:24,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:25,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:26,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:27,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:28,554 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:28,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:29,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:29,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:30,554 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:30,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:31,998 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:32,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:33,637 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:15:33,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:15:33,986 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:15:34,029 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:34,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:35,124 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:35,998 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:36,998 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:37,998 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:38,998 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:39,998 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:40,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:40,998 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:41,998 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:42,998 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:43,999 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:44,505 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:15:44,505 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_c93c245f94bccf988bf8.mp4 with policy now +2025-01-18 20:15:44,506 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:15:44,506 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:15:44,506 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:15:44,507 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:15:44,559 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:15:44,559 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:44,559 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/1_eval_1_c93c245f94bccf988bf8.mp4 +2025-01-18 20:15:44,559 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:15:44,767 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/b6ysw7lj-media/videos/eval/1_eval_1_c93c245f94bccf988bf8.mp4 +2025-01-18 20:15:44,999 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:45,028 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/n8yvhgw8-media/videos/eval/1_eval_1_c93c245f94bccf988bf8.mp4 +2025-01-18 20:15:46,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:46,518 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:46,559 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:47,002 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:48,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:48,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:15:48,994 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:15:49,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:50,002 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:50,561 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:51,002 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:51,953 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:52,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:52,562 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:53,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:54,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:55,006 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:56,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:56,563 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:15:57,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:57,218 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:57,391 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:15:57,391 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:15:57,392 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:15:57,563 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:15:58,004 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:58,564 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:59,004 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:00,004 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:01,004 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:02,004 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:02,333 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:02,565 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:03,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:03,638 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:16:03,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:16:03,986 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:16:04,029 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:05,004 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:06,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:07,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:08,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:08,075 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:09,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:10,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:11,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:12,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:13,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:13,906 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:14,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:15,006 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:16,006 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:16,570 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:17,006 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:18,006 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:18,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:16:18,986 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:16:19,029 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:19,082 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:20,006 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:21,006 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:22,007 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:23,007 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:24,007 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:24,992 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:25,007 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:26,007 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:27,007 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:28,008 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:28,201 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:16:28,201 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:16:28,202 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_35d91a735aea17ce0770.mp4 with policy now +2025-01-18 20:16:28,202 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:16:28,203 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:16:28,203 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:16:28,506 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/p1hyhc5j-media/videos/eval/2_eval_2_35d91a735aea17ce0770.mp4 +2025-01-18 20:16:28,574 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:16:28,574 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:28,574 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/2_eval_2_35d91a735aea17ce0770.mp4 +2025-01-18 20:16:28,574 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:16:28,848 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/nkyldyvs-media/videos/eval/2_eval_2_35d91a735aea17ce0770.mp4 +2025-01-18 20:16:29,009 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:29,999 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:30,009 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:30,574 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:31,011 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:32,012 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:33,014 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:33,639 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:16:33,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:16:33,991 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:16:34,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:34,576 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:35,009 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:35,661 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:36,009 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:36,577 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:37,014 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:38,011 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:39,014 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:40,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:40,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:41,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:42,013 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:42,579 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:16:42,832 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/ccky18u0-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:16:43,013 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:43,234 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:16:43,235 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:16:43,235 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:16:43,579 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:16:44,020 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:44,579 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:45,016 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:45,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:46,014 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:47,014 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:48,014 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:48,581 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:48,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:16:48,986 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:16:49,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:50,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:51,014 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:51,072 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:52,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:52,582 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:53,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:54,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:54,583 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:55,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:56,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:56,996 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:57,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:58,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:59,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:00,016 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:01,016 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:02,016 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:02,996 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:03,017 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:03,639 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:17:03,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:17:03,987 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:17:04,029 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:05,016 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:06,017 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:07,017 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:08,017 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:08,077 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:09,017 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:10,017 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:11,017 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:12,017 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:13,018 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:13,243 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:14,028 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:14,459 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:17:14,460 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c4ac9168429d53521dfd.mp4 with policy now +2025-01-18 20:17:14,462 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:17:14,463 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:17:14,463 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:17:14,463 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:17:14,590 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:17:14,590 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:17:14,590 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/3_eval_3_c4ac9168429d53521dfd.mp4 +2025-01-18 20:17:14,590 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:17:14,784 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/gfaw09av-media/videos/eval/3_eval_3_c4ac9168429d53521dfd.mp4 +2025-01-18 20:17:15,034 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:15,059 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/bra3uouf-media/videos/eval/3_eval_3_c4ac9168429d53521dfd.mp4 +2025-01-18 20:17:16,030 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:16,590 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:17:17,025 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:18,036 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:19,000 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:17:19,000 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:17:19,032 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:19,159 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:20,029 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:20,592 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:17:21,027 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:22,035 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:22,593 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:17:23,030 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:24,031 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:25,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:25,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:26,034 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:27,031 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:27,289 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:17:27,290 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:17:27,290 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:17:27,594 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:17:27,594 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:17:28,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:28,595 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:17:29,040 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:30,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:30,035 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:30,595 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:17:31,034 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:32,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:32,596 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:17:33,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:33,640 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:17:33,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:17:33,987 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:17:34,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:34,597 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:17:35,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:35,569 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:36,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:36,598 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:17:37,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:38,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:39,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:40,034 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:41,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:41,034 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:42,034 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:43,034 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:44,035 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:45,035 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:46,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:46,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:47,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:48,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:48,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:17:48,987 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:17:49,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:50,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:51,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:51,106 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:52,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:53,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:54,038 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:55,038 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:56,038 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:57,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:57,038 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:58,038 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:59,055 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:59,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:17:59,165 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:17:59,165 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_73cfbcf4697dd02a1704.mp4 with policy now +2025-01-18 20:17:59,166 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:17:59,166 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:17:59,167 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:17:59,424 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/nffdq5oc-media/videos/eval/4_eval_4_73cfbcf4697dd02a1704.mp4 +2025-01-18 20:17:59,606 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:17:59,606 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/4_eval_4_73cfbcf4697dd02a1704.mp4 +2025-01-18 20:17:59,606 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:17:59,877 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/8t0wjxsc-media/videos/eval/4_eval_4_73cfbcf4697dd02a1704.mp4 +2025-01-18 20:18:00,048 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:00,606 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:01,060 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:02,065 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:02,178 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:03,055 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:03,646 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:18:03,996 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:18:03,997 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:18:04,056 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:04,608 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:05,053 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:06,049 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:06,608 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:07,052 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:07,630 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:08,057 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:09,055 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:10,056 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:10,345 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:18:10,345 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:18:10,345 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:18:10,610 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:18:10,610 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:18:10,610 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:10,864 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/16d0k2cs-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:18:11,060 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:12,057 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:12,610 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:12,969 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:13,068 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:14,057 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:14,611 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:15,057 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:16,057 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:16,612 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:17,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:18,004 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:18,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:18,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:18:18,988 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:18:19,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:20,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:21,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:22,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:23,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:23,140 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:24,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:25,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:26,059 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:27,059 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:28,059 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:29,004 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:29,060 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:30,060 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:31,060 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:32,060 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:32,618 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:33,061 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:33,641 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:18:33,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:18:33,988 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:18:34,061 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:34,149 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:35,060 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:36,061 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:37,061 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:38,061 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:39,061 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:40,004 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:40,061 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:41,065 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:41,906 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:18:41,906 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_8eee7b4209c8cc001b29.mp4 with policy now +2025-01-18 20:18:41,909 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:18:41,909 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:18:41,910 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:18:41,911 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:18:42,065 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:42,166 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/tyzl746m-media/videos/eval/5_eval_5_8eee7b4209c8cc001b29.mp4 +2025-01-18 20:18:42,621 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:18:42,622 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:42,622 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/5_eval_5_8eee7b4209c8cc001b29.mp4 +2025-01-18 20:18:42,622 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:18:42,901 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/i811lut9-media/videos/eval/5_eval_5_8eee7b4209c8cc001b29.mp4 +2025-01-18 20:18:43,065 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:43,622 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:44,077 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:44,623 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:45,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:45,068 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:46,089 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:46,623 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:47,074 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:48,079 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:48,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:18:48,990 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:18:49,077 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:50,081 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:50,096 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:50,625 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:51,075 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:52,076 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:53,073 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:54,082 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:54,400 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:18:54,400 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:18:54,401 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:18:54,627 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:18:54,627 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:18:55,087 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:55,197 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:56,081 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:56,628 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:57,083 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:58,097 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:59,080 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:00,080 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:00,629 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:01,018 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:01,080 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:02,081 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:03,081 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:03,641 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:19:03,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:19:03,990 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:19:04,081 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:05,081 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:06,081 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:06,086 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:07,082 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:08,081 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:09,082 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:10,083 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:11,083 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:12,018 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:12,083 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:13,083 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:14,083 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:15,083 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:16,083 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:17,018 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:17,083 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:18,084 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:18,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:19:18,990 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:19:19,084 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:20,084 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:21,084 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:22,084 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:22,144 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:23,084 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:24,084 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:25,084 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:26,088 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:26,233 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:19:26,233 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_6f92e10ddc5700f95864.mp4 with policy now +2025-01-18 20:19:26,244 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:19:26,245 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:19:26,245 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:19:26,246 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:19:26,493 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/85f3h9co-media/videos/eval/6_eval_6_6f92e10ddc5700f95864.mp4 +2025-01-18 20:19:26,639 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:19:26,639 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:26,639 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/6_eval_6_6f92e10ddc5700f95864.mp4 +2025-01-18 20:19:26,639 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:19:26,921 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/kaay7jpj-media/videos/eval/6_eval_6_6f92e10ddc5700f95864.mp4 +2025-01-18 20:19:27,093 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:27,269 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:28,094 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:28,640 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:29,093 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:30,095 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:30,640 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:31,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:32,116 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:32,737 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:33,098 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:33,641 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:19:33,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:19:33,994 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:19:34,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:34,642 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:35,114 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:36,115 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:37,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:37,778 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:38,114 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:38,455 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:19:38,455 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:19:38,456 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:19:38,644 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:19:38,644 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:19:38,644 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:38,895 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/irpocnqm-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:19:39,118 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:40,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:40,645 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:41,118 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:42,116 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:42,645 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:43,122 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:43,678 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:44,116 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:44,647 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:45,116 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:46,116 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:46,647 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:47,116 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:48,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:48,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:19:48,990 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:19:49,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:49,145 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:50,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:51,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:52,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:53,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:54,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:55,024 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:55,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:56,118 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:57,118 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:58,118 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:59,118 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:00,024 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:00,118 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:01,118 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:02,118 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:03,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:03,642 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:20:03,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:20:03,990 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:20:04,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:05,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:05,152 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:06,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:07,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:08,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:09,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:10,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:10,725 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:11,141 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:12,084 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:20:12,084 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_ec916f9f1cb86bb268d4.mp4 with policy now +2025-01-18 20:20:12,091 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:20:12,092 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:20:12,092 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:20:12,092 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:20:12,129 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:12,350 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/zrcdmlmp-media/videos/eval/7_eval_7_ec916f9f1cb86bb268d4.mp4 +2025-01-18 20:20:12,658 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:20:12,658 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:12,658 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/7_eval_7_ec916f9f1cb86bb268d4.mp4 +2025-01-18 20:20:12,658 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:20:12,928 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/90kl4c8p-media/videos/eval/7_eval_7_ec916f9f1cb86bb268d4.mp4 +2025-01-18 20:20:13,148 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:13,659 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:14,141 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:14,659 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:15,139 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:15,752 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:16,156 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:16,660 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:17,131 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:18,148 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:18,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:20:18,991 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:20:19,155 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:20,150 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:20,508 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:20:20,508 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:20:20,522 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:20:20,661 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:20:20,662 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:20:20,662 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:20,861 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:21,176 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:22,159 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:22,662 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:23,176 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:24,161 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:24,663 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:25,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:26,034 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:26,174 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:27,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:28,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:28,665 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:29,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:30,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:30,665 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:31,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:32,029 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:32,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:33,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:33,642 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:20:33,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:20:33,991 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:20:34,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:35,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:36,164 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:37,127 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:37,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:38,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:39,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:40,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:41,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:42,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:43,030 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:43,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:44,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:45,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:46,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:47,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:48,030 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:48,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:48,991 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:20:48,991 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:20:49,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:50,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:51,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:52,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:53,108 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:53,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:54,180 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:54,674 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:54,929 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:20:54,929 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_c1b631f94f507d0b8029.mp4 with policy now +2025-01-18 20:20:54,943 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:20:54,944 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:20:54,944 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:20:54,944 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:20:55,172 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:55,190 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/mjyndchd-media/videos/eval/8_eval_8_c1b631f94f507d0b8029.mp4 +2025-01-18 20:20:55,675 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:20:55,675 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/8_eval_8_c1b631f94f507d0b8029.mp4 +2025-01-18 20:20:55,675 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:20:55,978 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/alo1tma0-media/videos/eval/8_eval_8_c1b631f94f507d0b8029.mp4 +2025-01-18 20:20:56,184 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:56,675 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:57,186 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:57,676 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:58,191 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:58,604 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:59,200 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:00,192 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:00,677 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:01,210 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:02,217 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:02,678 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:03,212 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:03,643 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:21:03,644 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:04,013 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:21:04,014 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:21:04,214 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:04,560 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:21:04,561 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:21:04,561 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:21:04,678 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:21:04,679 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:21:04,679 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:04,937 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/r2bwdo23-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:21:05,216 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:06,220 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:06,679 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:07,216 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:08,207 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:08,680 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:09,040 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:09,206 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:10,206 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:10,681 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:11,206 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:12,206 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:13,206 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:14,040 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:14,207 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:15,207 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:16,207 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:17,207 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:18,208 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:18,992 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:21:18,992 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:21:19,146 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:19,208 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:20,208 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:21,208 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:22,208 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:23,208 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:24,208 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:25,040 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:25,209 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:26,209 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:27,209 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:28,209 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:29,210 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:30,040 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:30,210 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:31,210 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:32,210 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:33,210 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:33,643 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:21:33,992 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:21:33,993 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:21:34,210 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:35,218 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:35,612 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:36,221 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:36,387 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:21:36,387 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_45f46258144727274eca.mp4 with policy now +2025-01-18 20:21:36,389 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:21:36,390 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:21:36,390 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:21:36,390 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:21:36,644 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/cylhyune-media/videos/eval/9_eval_9_45f46258144727274eca.mp4 +2025-01-18 20:21:36,692 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:21:36,692 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:36,692 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/9_eval_9_45f46258144727274eca.mp4 +2025-01-18 20:21:36,692 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:21:36,966 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/8vkplk5e-media/videos/eval/9_eval_9_45f46258144727274eca.mp4 +2025-01-18 20:21:37,219 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:37,692 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:38,237 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:38,692 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:39,238 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:39,693 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:40,241 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:40,693 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:41,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:41,245 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:42,238 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:42,694 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:43,225 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:44,237 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:44,695 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:45,246 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:46,238 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:46,351 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:46,612 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:21:46,613 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:21:46,613 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:21:46,696 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:21:46,696 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:21:47,248 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:48,244 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:48,696 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:49,019 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:21:49,019 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:21:49,242 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:50,258 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:51,252 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:51,473 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:52,251 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:52,698 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:53,275 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:54,267 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:55,257 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:56,257 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:56,699 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:57,045 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:57,257 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:58,257 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:59,257 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:00,258 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:01,258 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:02,059 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:02,258 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:03,258 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:03,644 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:22:03,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:22:03,998 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:22:04,258 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:05,259 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:06,259 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:07,156 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:07,259 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:08,259 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:09,259 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:10,259 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:11,259 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:12,259 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:12,706 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:12,841 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:13,260 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:14,260 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:15,260 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:16,260 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:16,708 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:17,260 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:18,242 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:18,260 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:18,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:22:18,998 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:22:19,261 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:20,260 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:21,261 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:22,262 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:22,485 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:22:22,485 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_dbfaa8aaf0c838051064.mp4 with policy now +2025-01-18 20:22:22,487 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:22:22,488 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:22:22,488 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:22:22,489 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:22:22,710 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:22:22,710 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:22,710 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/10_eval_10_dbfaa8aaf0c838051064.mp4 +2025-01-18 20:22:22,710 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:22:22,739 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/d4kkewji-media/videos/eval/10_eval_10_dbfaa8aaf0c838051064.mp4 +2025-01-18 20:22:23,033 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/434t8aip-media/videos/eval/10_eval_10_dbfaa8aaf0c838051064.mp4 +2025-01-18 20:22:23,281 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:23,548 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:24,268 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:24,711 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:25,284 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:25,711 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:26,292 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:27,293 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:28,310 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:28,712 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:29,077 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:29,287 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:30,300 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:30,713 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:31,315 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:32,302 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:33,320 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:33,644 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:22:34,024 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:22:34,024 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:22:34,168 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:34,317 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:35,310 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:36,323 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:37,338 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:38,329 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:39,327 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:40,096 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:40,332 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:41,336 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:42,330 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:43,337 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:44,337 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:45,364 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:46,095 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:46,333 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:47,344 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:48,341 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:48,686 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:22:48,686 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:22:48,686 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:22:48,720 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:22:48,720 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:22:48,720 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:48,993 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/8xpe97fa-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:22:49,017 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:22:49,018 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:22:49,354 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:50,359 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:50,721 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:51,264 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:51,360 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:52,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:53,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:54,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:54,722 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:55,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:56,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:57,075 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:57,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:58,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:59,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:00,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:01,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:02,075 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:02,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:03,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:03,645 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:23:04,001 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:23:04,002 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:23:04,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:05,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:06,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:07,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:07,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:08,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:09,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:10,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:11,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:12,358 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:13,076 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:13,358 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:14,358 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:14,730 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:15,358 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:16,358 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:17,358 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:18,076 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:18,358 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:19,002 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:23:19,002 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:23:19,359 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:20,360 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:20,733 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:20,733 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/11_eval_11_359c87ada34d682bae45.mp4 +2025-01-18 20:23:20,733 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:23:20,742 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:23:20,742 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_359c87ada34d682bae45.mp4 with policy now +2025-01-18 20:23:20,744 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:23:20,744 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:23:20,745 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:23:20,752 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:23:20,995 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/4vb69597-media/videos/eval/11_eval_11_359c87ada34d682bae45.mp4 +2025-01-18 20:23:21,251 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/li0z0nfk-media/videos/eval/11_eval_11_359c87ada34d682bae45.mp4 +2025-01-18 20:23:21,369 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:21,733 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:23:22,378 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:22,733 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:23,378 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:23,734 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:23,853 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:24,383 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:25,388 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:26,438 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:26,735 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:27,413 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:28,398 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:28,736 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:29,087 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:29,418 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:30,397 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:30,736 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:23:30,736 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:23:30,737 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:23:30,737 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:23:30,737 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:23:30,737 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:31,401 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:32,404 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:32,738 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:33,406 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:33,645 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:23:34,021 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:23:34,021 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:23:34,167 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:34,389 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:35,397 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:36,424 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:36,739 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:37,425 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:38,401 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:39,167 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:39,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:40,391 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:40,741 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:41,391 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:42,391 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:43,391 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:44,391 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:45,079 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:45,391 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:46,391 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:46,743 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:47,392 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:48,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:49,008 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:23:49,008 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:23:49,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:50,170 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:50,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:51,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:52,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:53,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:54,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:55,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:56,079 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:56,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:57,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:58,394 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:59,394 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:00,395 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:01,395 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:02,079 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:02,395 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:03,395 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:03,646 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:24:04,008 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:24:04,009 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:24:04,395 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:05,398 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:06,426 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:06,752 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:07,317 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:07,422 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:07,530 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:24:07,530 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_50069bff80de5f046230.mp4 with policy now +2025-01-18 20:24:07,539 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:24:07,539 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:24:07,539 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:24:07,539 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:24:07,752 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:24:07,752 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/12_eval_12_50069bff80de5f046230.mp4 +2025-01-18 20:24:07,752 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:24:07,787 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/xg2iymz1-media/videos/eval/12_eval_12_50069bff80de5f046230.mp4 +2025-01-18 20:24:08,044 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/7cxjiq6y-media/videos/eval/12_eval_12_50069bff80de5f046230.mp4 +2025-01-18 20:24:08,427 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:08,752 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:09,429 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:09,753 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:10,431 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:11,428 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:12,420 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:12,754 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:13,237 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:13,450 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:14,415 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:15,452 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:16,460 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:16,756 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:24:16,756 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:16,790 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:24:16,790 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:24:16,791 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:24:17,448 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:17,757 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:24:17,757 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:24:18,437 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:18,757 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:18,913 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:19,026 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:24:19,026 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:24:19,445 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:20,446 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:21,438 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:22,456 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:22,759 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:23,450 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:23,977 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:24,459 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:25,475 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:26,473 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:27,456 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:28,456 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:28,761 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:29,089 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:29,456 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:30,456 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:31,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:32,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:33,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:33,646 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:24:34,009 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:24:34,009 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:24:34,152 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:34,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:35,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:36,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:37,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:38,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:39,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:40,089 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:40,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:41,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:42,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:43,458 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:44,458 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:45,089 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:45,458 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:46,458 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:47,458 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:48,458 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:49,009 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:24:49,010 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:24:49,458 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:50,089 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:50,458 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:51,459 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:52,459 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:53,460 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:54,487 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:54,772 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:55,458 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:55,496 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:55,772 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/13_eval_13_114ad2ddaaeca4cacdd5.mp4 +2025-01-18 20:24:55,772 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:24:55,784 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:24:55,784 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_114ad2ddaaeca4cacdd5.mp4 with policy now +2025-01-18 20:24:55,799 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:24:55,799 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:24:55,799 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:24:55,799 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:24:56,052 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/zry9fzab-media/videos/eval/13_eval_13_114ad2ddaaeca4cacdd5.mp4 +2025-01-18 20:24:56,309 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/jsjkzokd-media/videos/eval/13_eval_13_114ad2ddaaeca4cacdd5.mp4 +2025-01-18 20:24:56,511 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:56,772 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:24:56,773 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:57,507 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:57,773 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:58,518 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:59,519 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:00,496 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:00,774 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:01,341 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:01,508 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:02,491 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:03,545 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:03,646 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:25:03,775 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:25:03,852 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:25:03,852 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:25:03,862 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:25:04,027 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/mdilzrm7-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:25:04,056 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:25:04,056 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:25:04,522 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:04,776 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:25:04,776 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:25:04,776 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:05,503 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:06,505 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:06,776 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:06,797 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:07,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:08,542 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:08,777 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:09,518 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:10,513 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:11,520 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:12,104 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:12,514 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:13,521 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:14,528 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:15,513 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:16,513 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:16,781 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:17,342 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:17,513 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:18,514 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:19,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:25:19,016 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:25:19,515 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:20,515 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:21,515 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:22,515 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:23,129 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:23,515 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:24,515 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:25,515 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:26,515 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:27,515 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:28,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:29,095 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:29,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:30,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:31,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:32,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:33,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:33,647 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:25:34,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:25:34,016 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:25:34,176 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:34,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:35,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:36,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:37,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:38,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:39,517 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:40,096 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:40,517 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:41,517 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:42,528 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:42,792 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:43,329 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:25:43,329 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_d43b1e73114608f44587.mp4 with policy now +2025-01-18 20:25:43,359 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:25:43,360 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:25:43,360 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:25:43,360 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:25:43,526 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:43,596 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/spkax966-media/videos/eval/14_eval_14_d43b1e73114608f44587.mp4 +2025-01-18 20:25:43,792 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:25:43,792 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/14_eval_14_d43b1e73114608f44587.mp4 +2025-01-18 20:25:43,792 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:25:44,065 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/ab6uze5e-media/videos/eval/14_eval_14_d43b1e73114608f44587.mp4 +2025-01-18 20:25:44,546 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:44,792 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:45,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:45,565 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:45,793 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:46,533 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:47,557 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:48,533 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:48,794 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:49,026 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:25:49,027 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:25:49,541 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:50,168 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:50,533 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:50,795 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:51,551 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:52,548 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:53,554 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:53,796 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:25:53,917 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:25:53,917 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:25:53,918 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:25:54,575 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:54,797 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:25:54,797 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:55,233 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:55,555 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:56,555 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:57,566 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:58,583 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:58,798 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:59,599 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:00,589 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:01,110 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:01,585 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:02,598 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:03,600 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:03,647 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:26:04,032 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:26:04,033 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:26:04,600 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:05,641 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:06,584 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:06,802 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:07,064 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:07,584 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:08,584 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:09,584 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:10,584 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:11,585 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:12,585 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:12,804 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:13,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:13,585 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:14,585 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:15,585 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:16,585 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:17,586 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:18,241 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:18,586 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:18,807 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:19,023 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:26:19,023 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:26:19,586 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:20,586 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:21,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:22,586 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:23,586 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:24,100 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:24,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:25,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:26,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:26,810 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:27,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:28,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:28,811 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:29,100 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:29,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:30,588 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:30,812 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:31,588 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:32,588 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:33,615 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:33,648 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:26:34,032 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:26:34,032 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:26:34,200 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:26:34,200 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:34,200 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_a134ab49305f64bfef91.mp4 with policy now +2025-01-18 20:26:34,202 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:26:34,203 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:26:34,203 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:26:34,203 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:26:34,470 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/lzxnvvl1-media/videos/eval/15_eval_15_a134ab49305f64bfef91.mp4 +2025-01-18 20:26:34,592 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:34,814 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:26:34,814 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:34,814 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/15_eval_15_a134ab49305f64bfef91.mp4 +2025-01-18 20:26:34,814 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:26:35,093 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/47xyupsk-media/videos/eval/15_eval_15_a134ab49305f64bfef91.mp4 +2025-01-18 20:26:35,621 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:35,814 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:36,624 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:37,650 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:38,618 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:38,815 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:39,636 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:40,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:40,626 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:41,637 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:42,624 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:43,646 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:43,818 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:44,670 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:45,125 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:45,638 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:46,665 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:47,675 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:48,663 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:49,060 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:26:49,060 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:26:49,673 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:49,820 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:26:49,986 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:26:49,986 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:26:49,986 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:26:50,666 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:50,821 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:26:50,821 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:26:50,922 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:51,671 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:51,821 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:52,692 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:53,686 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:53,822 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:54,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:55,694 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:55,823 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:56,131 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:56,692 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:57,695 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:58,726 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:59,715 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:00,745 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:01,142 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:01,722 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:02,720 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:03,648 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:27:03,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:03,826 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:04,025 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:27:04,025 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:27:04,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:05,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:05,827 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:06,187 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:06,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:07,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:08,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:09,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:10,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:11,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:12,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:12,699 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:13,699 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:14,699 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:15,699 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:16,699 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:17,699 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:18,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:18,699 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:19,025 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:27:19,025 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:27:19,699 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:20,700 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:21,700 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:22,700 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:23,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:23,700 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:24,700 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:25,700 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:25,836 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:26,701 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:27,701 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:28,701 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:29,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:29,701 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:30,725 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:30,974 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:27:30,974 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_fba70f3485b0b97a5e73.mp4 with policy now +2025-01-18 20:27:30,989 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:27:30,990 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:27:30,990 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:27:30,990 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:27:31,245 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/stvknmdz-media/videos/eval/16_eval_16_fba70f3485b0b97a5e73.mp4 +2025-01-18 20:27:31,743 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:31,838 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:27:31,838 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:31,839 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/16_eval_16_fba70f3485b0b97a5e73.mp4 +2025-01-18 20:27:31,839 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:27:32,130 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/quw091zt-media/videos/eval/16_eval_16_fba70f3485b0b97a5e73.mp4 +2025-01-18 20:27:32,742 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:33,649 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:27:33,731 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:33,839 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:34,039 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:27:34,040 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:27:34,180 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:34,734 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:35,760 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:36,749 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:37,733 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:37,841 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:38,770 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:39,762 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:39,842 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:40,051 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:40,778 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:40,842 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:27:41,050 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:27:41,050 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:27:41,050 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:27:41,102 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/52wmha3p-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:27:41,780 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:41,843 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:27:41,843 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:42,756 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:43,753 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:43,843 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:44,749 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:45,557 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:45,775 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:46,763 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:47,770 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:47,845 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:48,767 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:49,057 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:27:49,057 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:27:49,763 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:50,778 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:51,149 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:51,763 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:52,768 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:53,776 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:54,757 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:55,757 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:55,849 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:56,757 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:57,137 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:57,757 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:58,757 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:59,757 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:00,757 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:01,758 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:02,758 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:03,137 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:03,649 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:28:03,758 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:04,028 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:28:04,028 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:28:04,758 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:05,758 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:06,758 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:07,758 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:08,138 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:08,758 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:09,758 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:10,759 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:11,759 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:11,856 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:28:12,759 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:13,157 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:13,759 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:14,759 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:15,759 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:16,762 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:17,761 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:18,761 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:19,028 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:28:19,029 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:28:19,141 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:19,761 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:20,761 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:21,774 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:21,995 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:28:21,995 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_70585fb0ca24437a0f9f.mp4 with policy now +2025-01-18 20:28:22,026 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:28:22,026 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:28:22,026 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:28:22,027 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:28:22,292 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/89wxzpv4-media/videos/eval/17_eval_17_70585fb0ca24437a0f9f.mp4 +2025-01-18 20:28:22,786 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:22,861 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:28:22,861 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/17_eval_17_70585fb0ca24437a0f9f.mp4 +2025-01-18 20:28:22,861 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:28:23,409 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/h75uibxj-media/videos/eval/17_eval_17_70585fb0ca24437a0f9f.mp4 +2025-01-18 20:28:23,798 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:23,861 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:28:24,193 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:24,806 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:25,811 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:26,802 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:27,816 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:27,863 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:28:28,808 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:29,194 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:29,813 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:29,864 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:28:30,109 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:28:30,110 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:28:30,130 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:28:30,854 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:30,864 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:28:31,825 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:31,864 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:28:32,847 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:33,650 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:28:33,823 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:34,035 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:28:34,036 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:28:34,845 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:35,162 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:35,834 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:35,866 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:28:36,852 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:37,857 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:38,877 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:39,886 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:40,184 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:40,841 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:41,841 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:42,841 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:43,841 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:43,870 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:28:44,841 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:45,841 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:46,160 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:46,841 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:47,841 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:48,842 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:49,030 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:28:49,031 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:28:49,842 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:50,842 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:51,161 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:51,842 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:52,843 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:53,842 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:54,843 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:55,843 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:56,843 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:57,161 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:57,843 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:58,843 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:59,843 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:00,843 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:01,844 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:02,161 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:02,844 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:03,650 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:29:03,844 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:04,031 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:29:04,031 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:29:04,844 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:05,844 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:06,844 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:07,161 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:07,873 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:08,874 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:09,461 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:29:09,461 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_666566ff2e12fdecef30.mp4 with policy now +2025-01-18 20:29:09,510 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:29:09,511 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:29:09,511 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:29:09,512 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:29:09,723 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/5xldh2p6-media/videos/eval/18_eval_18_666566ff2e12fdecef30.mp4 +2025-01-18 20:29:09,857 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:09,881 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:29:09,881 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:29:09,881 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/18_eval_18_666566ff2e12fdecef30.mp4 +2025-01-18 20:29:09,881 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:29:10,160 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/xx9bayku-media/videos/eval/18_eval_18_666566ff2e12fdecef30.mp4 +2025-01-18 20:29:10,860 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:11,882 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:29:11,891 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:12,569 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:12,868 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:13,887 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:14,890 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:15,884 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:29:15,927 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:16,899 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:17,885 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:29:17,943 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:18,233 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:18,922 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:19,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:29:19,058 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:29:19,917 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:20,942 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:21,919 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:22,921 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:23,932 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:24,202 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:24,950 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:25,947 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:26,978 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:28,006 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:28,973 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:29,212 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:29,960 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:30,974 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:32,001 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:32,983 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:33,650 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:29:33,977 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:34,044 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:29:34,045 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:29:34,976 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:35,209 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:35,983 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:36,999 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:38,012 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:39,028 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:40,010 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:40,257 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:41,028 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:42,045 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:43,046 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:44,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:45,038 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:46,075 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:46,232 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:47,068 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:48,064 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:49,047 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:49,059 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:29:49,060 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:29:50,056 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:51,062 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:52,067 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:52,231 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:53,059 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:54,114 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:55,099 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:56,065 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:56,901 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:29:57,049 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:57,213 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:29:57,213 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:29:57,213 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:29:57,902 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:29:57,902 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:29:58,044 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:58,085 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:59,076 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:59,903 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:00,075 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:01,055 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:02,090 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:03,075 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:03,525 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:03,651 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:30:03,905 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:04,048 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:30:04,048 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:30:04,111 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:05,122 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:06,111 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:07,148 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:08,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:09,121 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:09,243 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:10,137 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:11,106 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:12,134 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:13,112 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:13,909 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:14,112 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:15,112 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:15,229 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:15,910 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:16,112 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:17,112 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:18,112 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:19,040 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:30:19,040 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:30:19,112 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:20,112 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:20,230 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:21,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:22,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:23,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:24,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:25,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:25,230 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:26,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:27,114 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:28,115 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:29,114 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:30,114 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:30,679 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:31,114 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:32,114 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:33,114 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:33,651 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:30:33,918 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:34,040 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:30:34,041 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:30:34,115 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:35,115 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:35,919 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:36,116 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:36,482 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:37,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:38,116 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:39,116 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:39,921 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:40,124 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:41,155 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:41,221 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:30:41,221 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:30:41,221 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_d8301c722455e230f538.mp4 with policy now +2025-01-18 20:30:41,222 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:30:41,222 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:41,223 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:41,504 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/eo65la2f-media/videos/eval/19_eval_19_d8301c722455e230f538.mp4 +2025-01-18 20:30:41,922 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:30:41,922 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:41,922 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/19_eval_19_d8301c722455e230f538.mp4 +2025-01-18 20:30:41,922 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:30:42,156 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:42,207 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/eglbeqc9-media/videos/eval/19_eval_19_d8301c722455e230f538.mp4 +2025-01-18 20:30:42,272 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:43,158 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:43,923 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:44,180 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:45,192 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:46,174 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:47,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:47,925 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:47,941 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:48,155 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:49,141 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:30:49,141 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:30:49,164 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:49,926 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:50,168 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:50,926 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:30:51,198 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:51,277 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:30:51,278 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:51,278 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:51,926 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:30:51,927 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:30:52,069 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 20:30:52,070 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 20:30:52,218 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:52,452 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:30:52,594 INFO wandb-upload_3:2569743 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpna8v3b00 +2025-01-18 20:30:52,853 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 20:30:52,863 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:30:52,927 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:30:52,927 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/table/eval/success_rate_ci_20_54e2754b6d7019322e44.table.json +2025-01-18 20:30:52,927 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/table/eval/total_success_rate_ci_21_84d6d84980b416ef739f.table.json +2025-01-18 20:30:52,928 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media +2025-01-18 20:30:52,928 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/table/eval +2025-01-18 20:30:52,928 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/table +2025-01-18 20:30:52,993 INFO SenderThread:2569743 [sender.py:send_request_log_artifact():1525] logged artifact run-h3eacbh9-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTk4MDY1MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzA3OTMxMQ==', 'latestArtifact': None}} +2025-01-18 20:30:52,993 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:30:52,993 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_54e2754b6d7019322e44.table.json with policy now +2025-01-18 20:30:52,993 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:30:52,993 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:52,994 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:52,994 DEBUG SenderThread:2569743 [sender.py:send():379] send: telemetry +2025-01-18 20:30:52,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:52,994 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:52,995 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:52,995 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:52,996 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:52,996 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:52,996 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:52,996 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:52,997 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:52,997 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:52,997 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:52,998 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:52,998 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:52,998 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:52,999 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:52,999 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:52,999 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:53,000 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:53,000 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:53,000 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:53,001 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:53,001 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:53,001 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:53,001 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 20:30:53,177 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:53,208 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/ey8we0uj-media/table/eval/total_success_rate_ci_21_84d6d84980b416ef739f.table.json +2025-01-18 20:30:53,235 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/lkkoy9fb-media/table/eval/success_rate_ci_20_54e2754b6d7019322e44.table.json +2025-01-18 20:30:53,261 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/v8391x8x-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:30:53,493 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/g0d69rld-media/table/eval/success_rate_ci_20_54e2754b6d7019322e44.table.json +2025-01-18 20:30:53,571 INFO wandb-upload_0:2569743 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpx2fn7a47 +2025-01-18 20:30:53,928 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:30:53,928 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:54,066 INFO SenderThread:2569743 [sender.py:send_request_log_artifact():1525] logged artifact run-h3eacbh9-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTk4MDY2Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzA3OTMxMw==', 'latestArtifact': None}} +2025-01-18 20:30:54,066 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:30:54,066 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_84d6d84980b416ef739f.table.json with policy now +2025-01-18 20:30:54,066 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:30:54,067 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:54,084 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:54,178 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:54,315 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/73vv4p4t-media/table/eval/total_success_rate_ci_21_84d6d84980b416ef739f.table.json +2025-01-18 20:30:54,928 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:30:55,178 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:55,929 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:56,178 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:57,178 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:58,178 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:58,245 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:59,178 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:00,178 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:01,178 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:02,179 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:03,179 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:03,245 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:31:03,652 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:31:04,050 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:31:04,050 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:31:04,179 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:05,179 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:06,179 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:07,179 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:08,179 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:08,245 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:31:09,179 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:10,180 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:11,180 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:12,180 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:13,180 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:14,180 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:14,245 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:31:15,093 DEBUG SenderThread:2569743 [sender.py:send():379] send: exit +2025-01-18 20:31:15,093 INFO SenderThread:2569743 [sender.py:send_exit():586] handling exit code: 0 +2025-01-18 20:31:15,094 INFO SenderThread:2569743 [sender.py:send_exit():588] handling runtime: 1031 +2025-01-18 20:31:15,094 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:31:15,094 INFO SenderThread:2569743 [sender.py:send_exit():594] send defer +2025-01-18 20:31:15,094 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:15,094 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-18 20:31:15,094 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:15,095 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-18 20:31:15,095 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 1 +2025-01-18 20:31:15,095 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:15,095 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-18 20:31:15,095 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:15,095 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-18 20:31:15,095 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 2 +2025-01-18 20:31:15,095 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:15,095 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-18 20:31:15,095 INFO HandlerThread:2569743 [system_monitor.py:finish():203] Stopping system monitor +2025-01-18 20:31:15,095 DEBUG SystemMonitor:2569743 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-18 20:31:15,095 DEBUG SystemMonitor:2569743 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-18 20:31:15,096 INFO HandlerThread:2569743 [interfaces.py:finish():200] Joined cpu monitor +2025-01-18 20:31:15,096 INFO HandlerThread:2569743 [interfaces.py:finish():200] Joined disk monitor +2025-01-18 20:31:15,164 INFO HandlerThread:2569743 [interfaces.py:finish():200] Joined gpu monitor +2025-01-18 20:31:15,165 INFO HandlerThread:2569743 [interfaces.py:finish():200] Joined memory monitor +2025-01-18 20:31:15,165 INFO HandlerThread:2569743 [interfaces.py:finish():200] Joined network monitor +2025-01-18 20:31:15,165 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:15,165 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-18 20:31:15,165 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 3 +2025-01-18 20:31:15,165 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:31:15,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:15,166 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-18 20:31:15,166 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:15,166 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-18 20:31:15,166 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 4 +2025-01-18 20:31:15,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:15,166 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-18 20:31:15,939 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:31:17,940 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:31:21,287 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:31:21,287 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:31:21,287 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: keepalive +2025-01-18 20:31:21,289 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:21,289 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-18 20:31:21,289 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 5 +2025-01-18 20:31:21,289 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:31:21,289 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:21,289 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-18 20:31:21,291 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:31:21,292 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:31:21,292 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:31:21,292 DEBUG SenderThread:2569743 [sender.py:send():379] send: summary +2025-01-18 20:31:21,293 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:31:21,293 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:21,293 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-18 20:31:21,293 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 6 +2025-01-18 20:31:21,293 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:21,293 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-18 20:31:21,293 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:21,293 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-18 20:31:21,297 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:31:21,426 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 7 +2025-01-18 20:31:21,426 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:21,426 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-18 20:31:21,426 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:21,426 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-18 20:31:21,942 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:31:21,942 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/config.yaml +2025-01-18 20:31:22,095 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:31:23,397 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 8 +2025-01-18 20:31:23,397 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:31:23,398 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:23,398 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-18 20:31:23,398 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:23,398 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-18 20:31:23,398 INFO SenderThread:2569743 [job_builder.py:build():440] Attempting to build job artifact +2025-01-18 20:31:23,400 INFO SenderThread:2569743 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-18 20:31:23,402 INFO SenderThread:2569743 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-18 20:31:23,416 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 9 +2025-01-18 20:31:23,416 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:23,416 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-18 20:31:23,416 DEBUG SenderThread:2569743 [sender.py:send():379] send: artifact +2025-01-18 20:31:23,631 INFO SenderThread:2569743 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1MTk1NDMyOQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1MTk1NDMyOQ==', 'versionIndex': 449}}} +2025-01-18 20:31:23,631 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:23,631 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-18 20:31:23,631 INFO SenderThread:2569743 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-18 20:31:23,944 INFO SenderThread:2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:31:23,944 INFO SenderThread:2569743 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files +2025-01-18 20:31:23,944 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/config.yaml config.yaml +2025-01-18 20:31:23,944 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:31:23,944 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json wandb-summary.json +2025-01-18 20:31:23,944 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/requirements.txt requirements.txt +2025-01-18 20:31:23,944 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-metadata.json wandb-metadata.json +2025-01-18 20:31:23,944 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 20:31:23,945 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log output.log +2025-01-18 20:31:23,946 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/diff.patch diff.patch +2025-01-18 20:31:23,950 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-18 20:31:23,950 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/19_eval_19_d8301c722455e230f538.mp4 media/videos/eval/19_eval_19_d8301c722455e230f538.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/16_eval_16_fba70f3485b0b97a5e73.mp4 media/videos/eval/16_eval_16_fba70f3485b0b97a5e73.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/12_eval_12_50069bff80de5f046230.mp4 media/videos/eval/12_eval_12_50069bff80de5f046230.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/9_eval_9_45f46258144727274eca.mp4 media/videos/eval/9_eval_9_45f46258144727274eca.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/8_eval_8_c1b631f94f507d0b8029.mp4 media/videos/eval/8_eval_8_c1b631f94f507d0b8029.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/4_eval_4_73cfbcf4697dd02a1704.mp4 media/videos/eval/4_eval_4_73cfbcf4697dd02a1704.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/18_eval_18_666566ff2e12fdecef30.mp4 media/videos/eval/18_eval_18_666566ff2e12fdecef30.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/6_eval_6_6f92e10ddc5700f95864.mp4 media/videos/eval/6_eval_6_6f92e10ddc5700f95864.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/5_eval_5_8eee7b4209c8cc001b29.mp4 media/videos/eval/5_eval_5_8eee7b4209c8cc001b29.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/17_eval_17_70585fb0ca24437a0f9f.mp4 media/videos/eval/17_eval_17_70585fb0ca24437a0f9f.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/11_eval_11_359c87ada34d682bae45.mp4 media/videos/eval/11_eval_11_359c87ada34d682bae45.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/13_eval_13_114ad2ddaaeca4cacdd5.mp4 media/videos/eval/13_eval_13_114ad2ddaaeca4cacdd5.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/7_eval_7_ec916f9f1cb86bb268d4.mp4 media/videos/eval/7_eval_7_ec916f9f1cb86bb268d4.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/2_eval_2_35d91a735aea17ce0770.mp4 media/videos/eval/2_eval_2_35d91a735aea17ce0770.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/15_eval_15_a134ab49305f64bfef91.mp4 media/videos/eval/15_eval_15_a134ab49305f64bfef91.mp4 +2025-01-18 20:31:23,952 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/14_eval_14_d43b1e73114608f44587.mp4 media/videos/eval/14_eval_14_d43b1e73114608f44587.mp4 +2025-01-18 20:31:23,952 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/10_eval_10_dbfaa8aaf0c838051064.mp4 media/videos/eval/10_eval_10_dbfaa8aaf0c838051064.mp4 +2025-01-18 20:31:23,952 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/1_eval_1_c93c245f94bccf988bf8.mp4 media/videos/eval/1_eval_1_c93c245f94bccf988bf8.mp4 +2025-01-18 20:31:23,952 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/3_eval_3_c4ac9168429d53521dfd.mp4 media/videos/eval/3_eval_3_c4ac9168429d53521dfd.mp4 +2025-01-18 20:31:23,952 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/0_eval_0_958873708057cb5c96d0.mp4 media/videos/eval/0_eval_0_958873708057cb5c96d0.mp4 +2025-01-18 20:31:23,952 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/table/eval/total_success_rate_ci_21_84d6d84980b416ef739f.table.json media/table/eval/total_success_rate_ci_21_84d6d84980b416ef739f.table.json +2025-01-18 20:31:23,952 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/table/eval/success_rate_ci_20_54e2754b6d7019322e44.table.json media/table/eval/success_rate_ci_20_54e2754b6d7019322e44.table.json +2025-01-18 20:31:23,952 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 10 +2025-01-18 20:31:23,952 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:23,952 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-18 20:31:23,952 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:23,953 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-18 20:31:23,953 INFO SenderThread:2569743 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 20:31:24,095 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:31:24,096 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:31:24,116 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:31:24,128 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/config.yaml +2025-01-18 20:31:24,136 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/requirements.txt +2025-01-18 20:31:24,186 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:31:24,387 INFO Thread-11 (_thread_body):2569743 [sender.py:transition_state():614] send defer: 11 +2025-01-18 20:31:24,387 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:24,387 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-18 20:31:24,387 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:24,387 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-18 20:31:24,387 INFO SenderThread:2569743 [file_pusher.py:join():175] waiting for file pusher +2025-01-18 20:31:24,390 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 12 +2025-01-18 20:31:24,391 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:24,391 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-18 20:31:24,391 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:24,391 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-18 20:31:24,391 INFO SenderThread:2569743 [file_stream.py:finish():601] file stream finish called +2025-01-18 20:31:24,585 INFO SenderThread:2569743 [file_stream.py:finish():605] file stream finish is done +2025-01-18 20:31:24,585 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 13 +2025-01-18 20:31:24,585 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:24,585 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-18 20:31:24,586 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:24,586 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-18 20:31:24,586 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 14 +2025-01-18 20:31:24,586 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:24,586 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-18 20:31:24,586 DEBUG SenderThread:2569743 [sender.py:send():379] send: final +2025-01-18 20:31:24,586 DEBUG SenderThread:2569743 [sender.py:send():379] send: footer +2025-01-18 20:31:24,586 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:24,586 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-18 20:31:24,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:31:24,587 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:31:24,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: server_info +2025-01-18 20:31:24,587 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: server_info +2025-01-18 20:31:24,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:31:24,589 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: get_summary +2025-01-18 20:31:24,589 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-18 20:31:24,590 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:24,645 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:31:24,645 INFO MainThread:2569743 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-18 20:31:24,646 INFO MainThread:2569743 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-18 20:31:24,646 INFO MainThread:2569743 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-18 20:31:24,646 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: shutdown +2025-01-18 20:31:24,646 INFO HandlerThread:2569743 [handler.py:finish():882] shutting down handler +2025-01-18 20:31:25,589 INFO WriterThread:2569743 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/run-h3eacbh9.wandb +2025-01-18 20:31:25,645 INFO SenderThread:2569743 [sender.py:finish():1615] shutting down sender +2025-01-18 20:31:25,645 INFO SenderThread:2569743 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 20:31:25,645 INFO SenderThread:2569743 [file_pusher.py:join():175] waiting for file pusher diff --git a/h3eacbh9/wandb/debug.log b/h3eacbh9/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e02dc0e2bc4f8b61dfcce249e1d4177d4286a569 --- /dev/null +++ b/h3eacbh9/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-18 20:14:03,259 INFO MainThread:2569579 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-18 20:14:03,259 INFO MainThread:2569579 [wandb_setup.py:_flush():76] Configure stats pid to 2569579 +2025-01-18 20:14:03,259 INFO MainThread:2569579 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-18 20:14:03,259 INFO MainThread:2569579 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/settings +2025-01-18 20:14:03,260 INFO MainThread:2569579 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-18 20:14:03,260 INFO MainThread:2569579 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-18 20:14:03,260 INFO MainThread:2569579 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-18 20:14:03,260 INFO MainThread:2569579 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-18 20:14:03,260 INFO MainThread:2569579 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/logs/debug.log +2025-01-18 20:14:03,260 INFO MainThread:2569579 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/logs/debug-internal.log +2025-01-18 20:14:03,260 INFO MainThread:2569579 [wandb_init.py:init():569] calling init triggers +2025-01-18 20:14:03,261 INFO MainThread:2569579 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_0', 'wandb_notes': '3465_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9'} +2025-01-18 20:14:03,261 INFO MainThread:2569579 [wandb_init.py:init():619] starting backend +2025-01-18 20:14:03,261 INFO MainThread:2569579 [wandb_init.py:init():623] setting up manager +2025-01-18 20:14:03,265 INFO MainThread:2569579 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-18 20:14:03,266 INFO MainThread:2569579 [wandb_init.py:init():631] backend started and connected +2025-01-18 20:14:03,278 INFO MainThread:2569579 [wandb_init.py:init():720] updated telemetry +2025-01-18 20:14:03,284 INFO MainThread:2569579 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-18 20:14:03,542 INFO MainThread:2569579 [wandb_run.py:_on_init():2435] communicating current version +2025-01-18 20:14:03,616 INFO MainThread:2569579 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-18 20:14:03,617 INFO MainThread:2569579 [wandb_init.py:init():804] starting run threads in backend +2025-01-18 20:14:03,985 INFO MainThread:2569579 [wandb_run.py:_console_start():2413] atexit reg +2025-01-18 20:14:03,985 INFO MainThread:2569579 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-18 20:14:03,985 INFO MainThread:2569579 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-18 20:14:03,985 INFO MainThread:2569579 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-18 20:14:03,987 INFO MainThread:2569579 [wandb_init.py:init():847] run started, returning control to user process +2025-01-18 20:14:03,987 INFO MainThread:2569579 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/tb, True +2025-01-18 20:14:09,077 INFO MainThread:2569579 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-18 20:31:25,647 WARNING MsgRouterThr:2569579 [router.py:message_loop():77] message_loop has been closed diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/code/FISH/eval_robot.py b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/config.yaml b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f298272c8f2e0d61e51239706345394621d77863 --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/config.yaml @@ -0,0 +1,927 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '3465_0' +wandb_notes: + desc: null + value: 3465_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: null +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 20 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737249243 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/diff.patch b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..f434f8847b435a60c5fa887dbb28833036f5dd51 --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/diff.patch @@ -0,0 +1,102 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0b377ae..03918f5 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -132,8 +132,18 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1000_0' # all + + # all books, 10/20 demos per book ++# crp_D+act history ++# wandb_run_id: '3456_0' ++# wandb_run_id: '3456_1' # seed 1 ++# wandb_run_id: '3456_2' # seed 2 ++ ++# all books, 06/20 demos per book ++# crp_D+act history ++wandb_run_id: '3465_0' ++# wandb_run_id: '3465_1' # seed 1 ++# wandb_run_id: '3465_2' # seed 2 ++ + # RGBD+mask+act history +-wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + +@@ -191,7 +201,7 @@ wandb_run_id: '1002_0' + true_action_history: false + wandb_notes: null + +-checkpoint_epoch: 12000 # for depth + all contact ++checkpoint_epoch: 7500 # for depth + all contact + load_residual_weight: false + + checkpoint_root_dir: '/home/${oc.env:USER}/fish_leon/FISH' +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..9509617 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,15 +167,21 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ # '3456_0', ++ # '3456_1', ++ # '3456_2', ++ '3465_0', ++ '3465_1', ++ '3465_2', + ] + +-checkpoint_epoch = 12000 ++checkpoint_epoch = 7500 + + # use subprocess to download the model checkpoints in parallel + process_list = [] +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/table/eval/success_rate_ci_20_54e2754b6d7019322e44.table.json b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/table/eval/success_rate_ci_20_54e2754b6d7019322e44.table.json new file mode 100644 index 0000000000000000000000000000000000000000..acbb78ec3fe866b83513ec66e64acef4c9f876a5 --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/table/eval/success_rate_ci_20_54e2754b6d7019322e44.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.9, 0.6478986143770058, 0.9880935138977903, 0.6879685222726633, 0.9747144355420543, 0.7650595455831652, 0.9736677668041369, 0.7752392216966422, 0.9721130783656747]]} \ No newline at end of file diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/table/eval/total_success_rate_ci_21_84d6d84980b416ef739f.table.json b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/table/eval/total_success_rate_ci_21_84d6d84980b416ef739f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..7fb6a731cb64627cf6011a786a6854505dd4d20f --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/table/eval/total_success_rate_ci_21_84d6d84980b416ef739f.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[39, 40, 0.975, 0.8683311640479618, 0.9995417957126389, 0.8927702902962017, 0.9965071956565379, 0.9260990469795616, 0.9941391329224076, 0.9064763336560784, 0.9959524566247081]]} \ No newline at end of file diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/0_eval_0_958873708057cb5c96d0.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/0_eval_0_958873708057cb5c96d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5da2dbdf8af84a1e2ae7e310e94e2a79fdf56309 --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/0_eval_0_958873708057cb5c96d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:958873708057cb5c96d07eec36f6c0e68759bdf4c854f4fae4331f95dc510299 +size 1142998 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/10_eval_10_dbfaa8aaf0c838051064.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/10_eval_10_dbfaa8aaf0c838051064.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bce106247fd8700eeb72e3df233bd6153c9dffba --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/10_eval_10_dbfaa8aaf0c838051064.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbfaa8aaf0c838051064d1f52334d6d83d61935c556bc18851fb329b8e51a97b +size 1290176 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/11_eval_11_359c87ada34d682bae45.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/11_eval_11_359c87ada34d682bae45.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..da117112871f053c9eca5be46c5831b36dc9b5ee --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/11_eval_11_359c87ada34d682bae45.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:359c87ada34d682bae4526c67a0e1660ebc14229bbfea9ebda9878bc7cbe49e2 +size 1165825 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/12_eval_12_50069bff80de5f046230.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/12_eval_12_50069bff80de5f046230.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..80264a1be329e3d40f7a32794abce6c751fbcf8e --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/12_eval_12_50069bff80de5f046230.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50069bff80de5f04623040da25c9abba38c19c8c6f485f7ecb364ef9c652a551 +size 1365675 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/13_eval_13_114ad2ddaaeca4cacdd5.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/13_eval_13_114ad2ddaaeca4cacdd5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..380b896adbd40c221465988facaa77cfede9743b --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/13_eval_13_114ad2ddaaeca4cacdd5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:114ad2ddaaeca4cacdd5ae91c8134dd40a3d5b8d16f5c1538175afff73d03d34 +size 1346252 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/14_eval_14_d43b1e73114608f44587.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/14_eval_14_d43b1e73114608f44587.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..494ea1cd16adaf7756f4448240608d5e53f89ccb --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/14_eval_14_d43b1e73114608f44587.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d43b1e73114608f44587bfbbd894e999f6f34aa1b616cbab8d4f16ce3f271adc +size 1413320 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/15_eval_15_a134ab49305f64bfef91.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/15_eval_15_a134ab49305f64bfef91.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9c219d50216d5ecaed401e2afe56d20990d9355e --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/15_eval_15_a134ab49305f64bfef91.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a134ab49305f64bfef917aadcc5002acbf1b6c847787e9add516af98661440cb +size 1461649 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/16_eval_16_fba70f3485b0b97a5e73.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/16_eval_16_fba70f3485b0b97a5e73.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..11fa3d1f2ebe734751ca63f5468d8fbfa80c246c --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/16_eval_16_fba70f3485b0b97a5e73.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fba70f3485b0b97a5e738e162bd29deed4662a79fc5279012753333773ee3b10 +size 1472547 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/17_eval_17_70585fb0ca24437a0f9f.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/17_eval_17_70585fb0ca24437a0f9f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..287815734eed0c3edcf3abe89a3d31753632fec9 --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/17_eval_17_70585fb0ca24437a0f9f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70585fb0ca24437a0f9f8d8ce89db96ec7d90d0e2abf545ad2ba5f0a1b9d31c9 +size 1504233 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/18_eval_18_666566ff2e12fdecef30.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/18_eval_18_666566ff2e12fdecef30.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..03ba6751ae466d8cacf2a9d82309f2e1678fdbbd --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/18_eval_18_666566ff2e12fdecef30.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:666566ff2e12fdecef308d665951ca56d112b39565b2350089304d70dea6847b +size 1399960 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/19_eval_19_d8301c722455e230f538.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/19_eval_19_d8301c722455e230f538.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e15b7c1d3d70e24137016d81d0c19d1556e4d6f5 --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/19_eval_19_d8301c722455e230f538.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8301c722455e230f538b18357aed48ab0500a9465cfbf6fbf0edd142fcf14c5 +size 1567147 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/1_eval_1_c93c245f94bccf988bf8.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/1_eval_1_c93c245f94bccf988bf8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..60db094cdc32c9ad320f819b51f1ec791a148f7f --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/1_eval_1_c93c245f94bccf988bf8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c93c245f94bccf988bf81c13b34f2b4f94a31c61b0b700a37cd4e15dacb444df +size 1151340 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/2_eval_2_35d91a735aea17ce0770.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/2_eval_2_35d91a735aea17ce0770.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b5cf0756fad4a918e8263bc2b1c22d1082f5d0f8 --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/2_eval_2_35d91a735aea17ce0770.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35d91a735aea17ce0770b204540d9606ca19ba47596be1fa84c03e8a76d48b12 +size 1160864 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/3_eval_3_c4ac9168429d53521dfd.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/3_eval_3_c4ac9168429d53521dfd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a923a768bccd7cc67a2904cac6051f895498872a --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/3_eval_3_c4ac9168429d53521dfd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4ac9168429d53521dfd398dfdf30167e676a6d017c37e6719de8c077ae2d6c5 +size 1157641 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/4_eval_4_73cfbcf4697dd02a1704.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/4_eval_4_73cfbcf4697dd02a1704.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5de845cb75f77a1083eb11c7bd1ac2d5c69f18b4 --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/4_eval_4_73cfbcf4697dd02a1704.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73cfbcf4697dd02a170455072cbbb3a1ca0352372cb4a0797e92ead11895af34 +size 1154605 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/5_eval_5_8eee7b4209c8cc001b29.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/5_eval_5_8eee7b4209c8cc001b29.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7ed8e7be99a9929e18fa2a68d5a9d82a13af8feb --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/5_eval_5_8eee7b4209c8cc001b29.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eee7b4209c8cc001b291cef3b235d9ad935ca7d79ef91da31d83f4b2565d510 +size 1146010 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/6_eval_6_6f92e10ddc5700f95864.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/6_eval_6_6f92e10ddc5700f95864.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d322d029bc8363ed25b74e667d4a2583292a3d2f --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/6_eval_6_6f92e10ddc5700f95864.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f92e10ddc5700f95864b3bd37bb543a48502145e8956f152881dac10ecc3e27 +size 1142252 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/7_eval_7_ec916f9f1cb86bb268d4.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/7_eval_7_ec916f9f1cb86bb268d4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7a131da3e5da8113f973d0ca4ac62ed5df185e93 --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/7_eval_7_ec916f9f1cb86bb268d4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec916f9f1cb86bb268d4b76b9be7a6f2b0e4f3512ce092e5d968e616457e41ba +size 1230184 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/8_eval_8_c1b631f94f507d0b8029.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/8_eval_8_c1b631f94f507d0b8029.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5a776ce207ea07d608e72c7c35ee5a73f965ecb8 --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/8_eval_8_c1b631f94f507d0b8029.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1b631f94f507d0b802907ef582d02c763dd6faa3b1b3b36bfa5e09e9ab836bc +size 1261905 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/9_eval_9_45f46258144727274eca.mp4 b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/9_eval_9_45f46258144727274eca.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..10236d1d5824622ddd2869d703ca5ddaade6c832 --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/9_eval_9_45f46258144727274eca.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45f46258144727274eca27c01c0f84d776e772cc618e870d73101f7b953894fb +size 1162867 diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..83e3df7bbc08396f8661116715f0c318a1eb5f8a --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log @@ -0,0 +1,1073 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737249249.078351]: resetting environment +[INFO] [1737249249.081686]: cleared current plan +[INFO] [1737249249.082591]: moving to home +[INFO] [1737249250.084410]: reached home +[INFO] [1737249250.085550]: reset action history +[INFO] [1737249252.793647]: environment reset +Starting episode 0 +[INFO] [1737249252.795943]: resetting environment +[INFO] [1737249252.799115]: cleared current plan +[INFO] [1737249252.799874]: moving to home +[INFO] [1737249253.801205]: reached home +[INFO] [1737249253.802403]: reset action history +[INFO] [1737249256.510297]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737249260.277192]: resetting environment +[INFO] [1737249260.282178]: cleared current plan +[INFO] [1737249260.282909]: moving to home +[INFO] [1737249261.284970]: reached home +[INFO] [1737249261.286748]: reset action history +[INFO] [1737249263.993652]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_0_2025-01-18-20-14-23.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:33905/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2569892] +process[print_text-2]: started with pid [2569893] +process[rosbag_record-3]: started with pid [2569894] +started bagging! +For topic gripper_width: timestamp difference is 103255429 for nearest: 1737249287602215209 - target: 1737249287498959780 at idx 249 +For topic gripper_width: timestamp difference is 103936218 for nearest: 1737249288402256273 - target: 1737249288298320055 at idx 249 +[INFO] [1737249291.002618]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1737249291.721252]: Stored episode 1. +[INFO] [1737249291.723065]: Saving video... +[INFO] [1737249291.906339]: Video saved! +Starting episode 1 +[INFO] [1737249291.917662]: resetting environment +[INFO] [1737249291.923811]: cleared current plan +[INFO] [1737249291.924057]: moving to home +[INFO] [1737249295.528689]: reached home +[INFO] [1737249295.529588]: reset action history +[INFO] [1737249298.235591]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737249312.410104]: resetting environment +[INFO] [1737249312.414311]: cleared current plan +[INFO] [1737249312.414557]: moving to home +[INFO] [1737249313.416325]: reached home +[INFO] [1737249313.416965]: reset action history +[INFO] [1737249316.127517]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_1_2025-01-18-20-15-16.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:45599/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2570003] +process[print_text-5]: started with pid [2570004] +process[rosbag_record-6]: started with pid [2570005] +started bagging! +For topic gripper_width: timestamp difference is 101145043 for nearest: 1737249326102257027 - target: 1737249326001111984 at idx 135 +For topic gripper_width: timestamp difference is 103432624 for nearest: 1737249328902255504 - target: 1737249328798822880 at idx 168 +[INFO] [1737249343.082117]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1737249343.853297]: Stored episode 2. +[INFO] [1737249343.853598]: Saving video... +[INFO] [1737249344.483072]: Video saved! +Starting episode 2 +[INFO] [1737249344.509384]: resetting environment +[INFO] [1737249344.516862]: cleared current plan +[INFO] [1737249344.517255]: moving to home +[WARN] [1737249344.460084]: Plan exhausted +[WARN] [1737249344.500313]: Plan exhausted +[INFO] [1737249348.126248]: reached home +[INFO] [1737249348.129300]: reset action history +[INFO] [1737249350.905746]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737249356.207621]: resetting environment +[INFO] [1737249356.214602]: cleared current plan +[INFO] [1737249356.215153]: moving to home +[INFO] [1737249357.216676]: reached home +[INFO] [1737249357.217611]: reset action history +[INFO] [1737249359.931995]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_2_2025-01-18-20-15-59.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:35953/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2570150] +process[print_text-8]: started with pid [2570151] +process[rosbag_record-9]: started with pid [2570152] +started bagging! +For topic gripper_width: timestamp difference is -104719336 for nearest: 1737249361102257793 - target: 1737249361206977129 at idx 17 +For topic gripper_width: timestamp difference is 106498073 for nearest: 1737249373902255367 - target: 1737249373795757294 at idx 186 +[INFO] [1737249386.877177]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit[pub_text-7] killing on exit +[INFO] [1737249387.649857]: Stored episode 3. +[INFO] [1737249387.652812]: Saving video... +[INFO] [1737249388.157207]: Video saved! +Starting episode 3 +[INFO] [1737249388.201803]: resetting environment +[INFO] [1737249388.217727]: cleared current plan +[INFO] [1737249388.221337]: moving to home +[WARN] [1737249388.194054]: Plan exhausted +[INFO] [1737249391.837249]: reached home +[INFO] [1737249391.840464]: reset action history +[INFO] [1737249394.617997]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737249402.357704]: resetting environment +[INFO] [1737249402.368201]: cleared current plan +[INFO] [1737249402.371972]: moving to home +[INFO] [1737249403.375507]: reached home +[INFO] [1737249403.376367]: reset action history +[INFO] [1737249406.104759]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_3_2025-01-18-20-16-46.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:42875/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2570319] +process[print_text-11]: started with pid [2570320] +process[rosbag_record-12]: started with pid [2570321] +started bagging! +For topic gripper_width: timestamp difference is 103222274 for nearest: 1737249411068951272 - target: 1737249410965728998 at idx 68 +For topic gripper_width: timestamp difference is 104511113 for nearest: 1737249412668950887 - target: 1737249412564439774 at idx 89 +[INFO] [1737249433.242801]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1737249434.023764]: Stored episode 4. +[INFO] [1737249434.032627]: Saving video... +[INFO] [1737249434.391923]: Video saved! +Starting episode 4 +[INFO] [1737249434.464583]: resetting environment +[INFO] [1737249434.477860]: cleared current plan +[INFO] [1737249434.478156]: moving to home +[INFO] [1737249438.081651]: reached home +[INFO] [1737249438.089238]: reset action history +[INFO] [1737249440.867763]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737249447.226007]: resetting environment +[INFO] [1737249447.238797]: cleared current plan +[INFO] [1737249447.245024]: moving to home +[INFO] [1737249448.248605]: reached home +[INFO] [1737249448.252421]: reset action history +[INFO] [1737249450.976738]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_4_2025-01-18-20-17-30.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:41281/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2570466] +process[print_text-14]: started with pid [2570467] +process[rosbag_record-15]: started with pid [2570468] +started bagging! +For topic gripper_width: timestamp difference is 107565262 for nearest: 1737249452703511097 - target: 1737249452595945835 at idx 25 +For topic gripper_width: timestamp difference is 104750887 for nearest: 1737249454502206579 - target: 1737249454397455692 at idx 44 +[INFO] [1737249478.081506]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1737249478.885342]: Stored episode 5. +[INFO] [1737249478.893093]: Saving video... +[INFO] [1737249479.122539]: Video saved! +Starting episode 5 +[INFO] [1737249479.168542]: resetting environment +[INFO] [1737249479.177425]: cleared current plan +[INFO] [1737249479.177719]: moving to home +[INFO] [1737249482.788205]: reached home +[INFO] [1737249482.793716]: reset action history +[INFO] [1737249485.587038]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737249489.879457]: resetting environment +[INFO] [1737249489.908102]: cleared current plan +[INFO] [1737249489.911692]: moving to home +[INFO] [1737249490.921672]: reached home +[INFO] [1737249490.931717]: reset action history +[INFO] [1737249493.709894]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_5_2025-01-18-20-18-13.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:42743/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-16]: started with pid [2570574] +process[print_text-17]: started with pid [2570575] +process[rosbag_record-18]: started with pid [2570599] +started bagging! +For topic gripper_width: timestamp difference is -116683671 for nearest: 1737249510102283528 - target: 1737249510218967199 at idx 222 +[INFO] [1737249520.661249]: Storing episode... +[rosbag_record-18] killing on exit +[print_text-17] killing on exit +[pub_text-16] killing on exit +[INFO] [1737249521.465914]: Stored episode 6. +[INFO] [1737249521.471130]: Saving video... +[INFO] [1737249521.859950]: Video saved! +[WARN] [1737249521.833997]: Plan exhausted +[WARN] [1737249521.875161]: Plan exhausted +[WARN] [1737249521.925362]: Plan exhausted +Starting episode 6 +[INFO] [1737249521.911885]: resetting environment +[INFO] [1737249521.934983]: cleared current plan +[INFO] [1737249521.937726]: moving to home +[INFO] [1737249525.548209]: reached home +[INFO] [1737249525.555216]: reset action history +[INFO] [1737249528.398367]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737249534.140478]: resetting environment +[INFO] [1737249534.159164]: cleared current plan +[INFO] [1737249534.178123]: moving to home +[INFO] [1737249535.186690]: reached home +[INFO] [1737249535.206267]: reset action history +[INFO] [1737249538.054020]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_6_2025-01-18-20-18-58.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:43607/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-19]: started with pid [2570719] +process[print_text-20]: started with pid [2570720] +process[rosbag_record-21]: started with pid [2570744] +started bagging! +[INFO] [1737249565.024206]: Storing episode... +[rosbag_record-21] killing on exit +[print_text-20] killing on exit +[pub_text-19] killing on exit +[INFO] [1737249565.837162]: Stored episode 7. +[INFO] [1737249565.846851]: Saving video... +[INFO] [1737249566.153891]: Video saved! +Starting episode 7 +[INFO] [1737249566.249096]: resetting environment +[INFO] [1737249566.268232]: cleared current plan +[INFO] [1737249566.268863]: moving to home +[INFO] [1737249569.879431]: reached home +[INFO] [1737249569.888703]: reset action history +[INFO] [1737249572.728491]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737249577.822815]: resetting environment +[INFO] [1737249577.853083]: cleared current plan +[INFO] [1737249577.853568]: moving to home +[INFO] [1737249578.860106]: reached home +[INFO] [1737249578.865837]: reset action history +[INFO] [1737249581.653394]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_7_2025-01-18-20-19-41.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:46327/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-22]: started with pid [2570885] +process[print_text-23]: started with pid [2570886] +process[rosbag_record-24]: started with pid [2570887] +started bagging! +[INFO] [1737249610.724713]: Storing episode... +[print_text-23] killing on exit +[rosbag_record-24] killing on exit +[pub_text-22] killing on exit +[INFO] [1737249611.501677]: Stored episode 8. +[INFO] [1737249611.513477]: Saving video... +[WARN] [1737249612.080686]: Plan exhausted +[WARN] [1737249612.113993]: Plan exhausted +[INFO] [1737249612.022954]: Video saved! +Starting episode 8 +[INFO] [1737249612.095842]: resetting environment +[INFO] [1737249612.137556]: cleared current plan +[INFO] [1737249612.142487]: moving to home +[INFO] [1737249615.750488]: reached home +[INFO] [1737249615.754663]: reset action history +[INFO] [1737249618.589978]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737249619.782883]: resetting environment +[INFO] [1737249619.813584]: cleared current plan +[INFO] [1737249619.836053]: moving to home +[INFO] [1737249620.857621]: reached home +[INFO] [1737249620.863363]: reset action history +[INFO] [1737249623.725026]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_8_2025-01-18-20-20-23.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:37223/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-25]: started with pid [2571031] +process[print_text-26]: started with pid [2571032] +process[rosbag_record-27]: started with pid [2571033] +started bagging! +For topic gripper_width: timestamp difference is 111919664 for nearest: 1737249628302210830 - target: 1737249628190291166 at idx 62 +[INFO] [1737249653.490227]: Storing episode... +[rosbag_record-27] killing on exit +[print_text-26] killing on exit +[pub_text-25] killing on exit +[INFO] [1737249654.331655]: Stored episode 9. +[INFO] [1737249654.333966]: Saving video... +[INFO] [1737249654.871116]: Video saved! +Starting episode 9 +[INFO] [1737249654.947713]: resetting environment +[INFO] [1737249654.984842]: cleared current plan +[INFO] [1737249654.988328]: moving to home +[WARN] [1737249654.803537]: Plan exhausted +[WARN] [1737249654.835123]: Plan exhausted +[WARN] [1737249654.883613]: Plan exhausted +[WARN] [1737249654.914959]: Plan exhausted +[WARN] [1737249654.955039]: Plan exhausted +[INFO] [1737249658.603506]: reached home +[INFO] [1737249658.616019]: reset action history +[INFO] [1737249661.519217]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737249663.787530]: resetting environment +[INFO] [1737249663.830411]: cleared current plan +[INFO] [1737249663.846425]: moving to home +[INFO] [1737249664.852948]: reached home +[INFO] [1737249664.873856]: reset action history +[INFO] [1737249667.798438]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_9_2025-01-18-20-21-07.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:43955/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-28]: started with pid [2571140] +process[print_text-29]: started with pid [2571141] +process[rosbag_record-30]: started with pid [2571165] +started bagging! +[INFO] [1737249694.791401]: Storing episode... +[rosbag_record-30] killing on exit +[pub_text-28] killing on exit +[print_text-29] killing on exit +[INFO] [1737249695.611362]: Stored episode 10. +[INFO] [1737249695.612773]: Saving video... +[WARN] [1737249696.119592]: Plan exhausted +[WARN] [1737249696.156100]: Plan exhausted +[WARN] [1737249696.197002]: Plan exhausted +[WARN] [1737249696.234255]: Plan exhausted +[WARN] [1737249696.274335]: Plan exhausted +[INFO] [1737249696.285563]: Video saved! +Starting episode 10 +[INFO] [1737249696.394614]: resetting environment +[INFO] [1737249696.422263]: cleared current plan +[INFO] [1737249696.438834]: moving to home +[WARN] [1737249696.315811]: Plan exhausted +[WARN] [1737249696.355062]: Plan exhausted +[WARN] [1737249696.399369]: Plan exhausted +[INFO] [1737249700.069276]: reached home +[INFO] [1737249700.080951]: reset action history +[INFO] [1737249703.085202]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737249706.422301]: resetting environment +[INFO] [1737249706.484681]: cleared current plan +[INFO] [1737249706.490438]: moving to home +[INFO] [1737249707.509834]: reached home +[INFO] [1737249707.512594]: reset action history +[INFO] [1737249710.427734]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_10_2025-01-18-20-21-50.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:32901/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-31]: started with pid [2571308] +process[print_text-32]: started with pid [2571309] +process[rosbag_record-33]: started with pid [2571310] +started bagging! +For topic gripper_width: timestamp difference is -117181696 for nearest: 1737249715802253328 - target: 1737249715919435024 at idx 74 +For topic gripper_width: timestamp difference is -104539193 for nearest: 1737249731602257692 - target: 1737249731706796885 at idx 249 +For topic gripper_width: timestamp difference is -102639147 for nearest: 1737249734002215914 - target: 1737249734104855061 at idx 249 +[INFO] [1737249741.170287]: Storing episode... +[rosbag_record-33] killing on exit +[pub_text-31] killing on exit[print_text-32] killing on exit +[INFO] [1737249741.965252]: Stored episode 11. +[INFO] [1737249741.979902]: Saving video... +[INFO] [1737249742.383360]: Video saved! +Starting episode 11 +[INFO] [1737249742.506773]: resetting environment +[INFO] [1737249742.542360]: cleared current plan +[INFO] [1737249742.547179]: moving to home +[WARN] [1737249742.515039]: Plan exhausted +[INFO] [1737249746.174920]: reached home +[INFO] [1737249746.184056]: reset action history +[INFO] [1737249749.208500]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737249768.161884]: resetting environment +[INFO] [1737249768.219151]: cleared current plan +[INFO] [1737249768.227497]: moving to home +[INFO] [1737249769.247456]: reached home +[INFO] [1737249769.259164]: reset action history +[INFO] [1737249772.267240]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_11_2025-01-18-20-22-52.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:40809/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-34]: started with pid [2571454] +process[print_text-35]: started with pid [2571455] +process[rosbag_record-36]: started with pid [2571456] +started bagging! +For topic gripper_width: timestamp difference is -250694015 for nearest: 1737249792902288458 - target: 1737249793152982473 at idx 249 +[INFO] [1737249799.412887]: Storing episode... +[print_text-35] killing on exit +[rosbag_record-36] killing on exit +[pub_text-34] killing on exit +[INFO] [1737249800.268633]: Stored episode 12. +[INFO] [1737249800.276276]: Saving video... +[INFO] [1737249800.631548]: Video saved! +Starting episode 12 +[INFO] [1737249800.775036]: resetting environment +[INFO] [1737249800.820366]: cleared current plan +[INFO] [1737249800.842957]: moving to home +[WARN] [1737249800.758654]: Plan exhausted +[WARN] [1737249800.799530]: Plan exhausted +[INFO] [1737249804.500443]: reached home +[INFO] [1737249804.509828]: reset action history +[INFO] [1737249807.575749]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737249810.100692]: resetting environment +[INFO] [1737249810.164159]: cleared current plan +[INFO] [1737249810.172378]: moving to home +[INFO] [1737249811.188725]: reached home +[INFO] [1737249811.205991]: reset action history +[INFO] [1737249814.218651]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_12_2025-01-18-20-23-34.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:45755/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-37]: started with pid [2571599] +process[print_text-38]: started with pid [2571600] +process[rosbag_record-39]: started with pid [2571601] +started bagging! +For topic gripper_width: timestamp difference is 104153316 for nearest: 1737249825202224414 - target: 1737249825098071098 at idx 154 +[INFO] [1737249845.960588]: Storing episode... +[print_text-38] killing on exit +[pub_text-37] killing on exit +[rosbag_record-39] killing on exit +[INFO] [1737249846.824931]: Stored episode 13. +[INFO] [1737249846.849400]: Saving video... +[INFO] [1737249847.394742]: Video saved! +Starting episode 13 +[INFO] [1737249847.546093]: resetting environment +[INFO] [1737249847.583766]: cleared current plan +[INFO] [1737249847.584242]: moving to home +[WARN] [1737249847.314416]: Plan exhausted +[WARN] [1737249847.360189]: Plan exhausted +[WARN] [1737249847.404541]: Plan exhausted +[WARN] [1737249847.436775]: Plan exhausted +[WARN] [1737249847.474025]: Plan exhausted +[WARN] [1737249847.515119]: Plan exhausted +[WARN] [1737249847.555854]: Plan exhausted +[INFO] [1737249851.187426]: reached home +[INFO] [1737249851.194680]: reset action history +[INFO] [1737249854.351411]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737249856.773642]: resetting environment +[INFO] [1737249856.850566]: cleared current plan +[INFO] [1737249856.855140]: moving to home +[INFO] [1737249857.867207]: reached home +[INFO] [1737249857.879064]: reset action history +[INFO] [1737249860.927919]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_13_2025-01-18-20-24-20.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:46207/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-40]: started with pid [2571742] +process[print_text-41]: started with pid [2571743] +process[rosbag_record-42]: started with pid [2571744] +started bagging! +[INFO] [1737249894.095142]: Storing episode... +[rosbag_record-42] killing on exit +[print_text-41] killing on exit +[pub_text-40] killing on exit +[INFO] [1737249894.998557]: Stored episode 14. +[INFO] [1737249895.020291]: Saving video... +[INFO] [1737249895.690021]: Video saved! +Starting episode 14 +[INFO] [1737249895.827899]: resetting environment +[INFO] [1737249895.857646]: cleared current plan +[INFO] [1737249895.865183]: moving to home +[WARN] [1737249895.447102]: Plan exhausted +[WARN] [1737249895.481983]: Plan exhausted +[WARN] [1737249895.514096]: Plan exhausted +[WARN] [1737249895.554084]: Plan exhausted +[WARN] [1737249895.594716]: Plan exhausted +[WARN] [1737249895.634303]: Plan exhausted +[WARN] [1737249895.701651]: Plan exhausted +[WARN] [1737249895.742103]: Plan exhausted +[WARN] [1737249895.763084]: Plan exhausted +[WARN] [1737249895.794225]: Plan exhausted +[WARN] [1737249895.833979]: Plan exhausted +[INFO] [1737249899.287156]: reached home +[INFO] [1737249899.314451]: reset action history +[INFO] [1737249902.381316]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737249903.655859]: resetting environment +[INFO] [1737249903.701357]: cleared current plan +[INFO] [1737249903.723204]: moving to home +[INFO] [1737249904.751634]: reached home +[INFO] [1737249904.755461]: reset action history +[INFO] [1737249907.814928]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_14_2025-01-18-20-25-07.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:36397/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-43]: started with pid [2571874] +process[print_text-44]: started with pid [2571875] +process[rosbag_record-45]: started with pid [2571876] +started bagging! +[INFO] [1737249942.053326]: Storing episode... +[pub_text-43] killing on exit +[rosbag_record-45] killing on exit +[print_text-44] killing on exit +[INFO] [1737249942.968349]: Stored episode 15. +[INFO] [1737249942.981990]: Saving video... +[INFO] [1737249943.223490]: Video saved! +Starting episode 15 +[INFO] [1737249943.374535]: resetting environment +[INFO] [1737249943.398065]: cleared current plan +[INFO] [1737249943.400188]: moving to home +[WARN] [1737249943.394072]: Plan exhausted +[INFO] [1737249947.004708]: reached home +[INFO] [1737249947.018468]: reset action history +[INFO] [1737249950.157722]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737249953.045297]: resetting environment +[INFO] [1737249953.148682]: cleared current plan +[INFO] [1737249953.180773]: moving to home +[INFO] [1737249954.187486]: reached home +[INFO] [1737249954.206108]: reset action history +[INFO] [1737249957.333216]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_15_2025-01-18-20-25-57.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:46411/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-46]: started with pid [2572019] +process[print_text-47]: started with pid [2572020] +process[rosbag_record-48]: started with pid [2572021] +started bagging! +For topic gripper_width: timestamp difference is -111905217 for nearest: 1737249970202275634 - target: 1737249970314180851 at idx 185 +For topic gripper_width: timestamp difference is -110679942 for nearest: 1737249971802249354 - target: 1737249971912929296 at idx 204 +For topic gripper_width: timestamp difference is -106465748 for nearest: 1737249977002252170 - target: 1737249977108717918 at idx 249 +For topic gripper_width: timestamp difference is -111908973 for nearest: 1737249984602250993 - target: 1737249984714159966 at idx 249 +For topic gripper_width: timestamp difference is -110258335 for nearest: 1737249985402251726 - target: 1737249985512510061 at idx 249 +For topic gripper_width: timestamp difference is -107800254 for nearest: 1737249986602252713 - target: 1737249986710052967 at idx 249 +For topic gripper_width: timestamp difference is -102472680 for nearest: 1737249989202282769 - target: 1737249989304755449 at idx 249 +For topic gripper_width: timestamp difference is -100843538 for nearest: 1737249990002251040 - target: 1737249990103094578 at idx 249 +[INFO] [1737249992.768161]: Storing episode... +[print_text-47] killing on exit +[pub_text-46] killing on exit +[rosbag_record-48] killing on exit +[INFO] [1737249993.562671]: Stored episode 16. +[INFO] [1737249993.586818]: Saving video... +[INFO] [1737249994.068738]: Video saved! +Starting episode 16 +[INFO] [1737249994.215227]: resetting environment +[INFO] [1737249994.237153]: cleared current plan +[INFO] [1737249994.242860]: moving to home +[WARN] [1737249994.125389]: Plan exhausted +[WARN] [1737249994.171160]: Plan exhausted +[WARN] [1737249994.204106]: Plan exhausted +[WARN] [1737249994.234037]: Plan exhausted +[INFO] [1737249997.845711]: reached home +[INFO] [1737249997.851232]: reset action history +[INFO] [1737250000.971283]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737250009.756773]: resetting environment +[INFO] [1737250009.885814]: cleared current plan +[INFO] [1737250009.905199]: moving to home +[INFO] [1737250010.921729]: reached home +[INFO] [1737250010.931715]: reset action history +[INFO] [1737250014.032817]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_16_2025-01-18-20-26-54.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:38803/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-49]: started with pid [2572170] +process[print_text-50]: started with pid [2572171] +process[rosbag_record-51]: started with pid [2572172] +started bagging! +For topic gripper_width: timestamp difference is -252673877 for nearest: 1737250043502208459 - target: 1737250043754882336 at idx 249 +[INFO] [1737250049.618902]: Storing episode... +[rosbag_record-51] killing on exit +[pub_text-49] killing on exit +[print_text-50] killing on exit +[INFO] [1737250050.445675]: Stored episode 17. +[INFO] [1737250050.452389]: Saving video... +[INFO] [1737250050.896428]: Video saved! +Starting episode 17 +[INFO] [1737250051.017439]: resetting environment +[INFO] [1737250051.073379]: cleared current plan +[INFO] [1737250051.081545]: moving to home +[WARN] [1737250050.953995]: Plan exhausted +[WARN] [1737250050.994015]: Plan exhausted +[WARN] [1737250051.038704]: Plan exhausted +[INFO] [1737250054.720576]: reached home +[INFO] [1737250054.750926]: reset action history +[INFO] [1737250057.972692]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737250060.220490]: resetting environment +[INFO] [1737250060.304836]: cleared current plan +[INFO] [1737250060.335065]: moving to home +[INFO] [1737250061.361071]: reached home +[INFO] [1737250061.364800]: reset action history +[INFO] [1737250064.500025]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_17_2025-01-18-20-27-44.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:40517/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-52]: started with pid [2572317] +process[print_text-53]: started with pid [2572318] +process[rosbag_record-54]: started with pid [2572319] +started bagging! +For topic gripper_width: timestamp difference is -116212932 for nearest: 1737250089102224978 - target: 1737250089218437910 at idx 249 +For topic gripper_width: timestamp difference is -114474609 for nearest: 1737250089902320549 - target: 1737250090016795158 at idx 249 +[INFO] [1737250100.702702]: Storing episode... +[rosbag_record-54] killing on exit +[pub_text-52] killing on exit +[print_text-53] killing on exit +[INFO] [1737250101.574209]: Stored episode 18. +[INFO] [1737250101.612487]: Saving video... +[INFO] [1737250101.838272]: Video saved! +Starting episode 18 +[INFO] [1737250102.037239]: resetting environment +[INFO] [1737250102.108043]: cleared current plan +[INFO] [1737250102.111111]: moving to home +[WARN] [1737250102.043149]: Plan exhausted +[WARN] [1737250102.078946]: Plan exhausted +[INFO] [1737250105.717511]: reached home +[INFO] [1737250105.752929]: reset action history +[INFO] [1737250108.924030]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737250109.208364]: resetting environment +[INFO] [1737250109.211081]: cleared current plan +[INFO] [1737250109.211239]: moving to home +[INFO] [1737250110.213425]: reached home +[INFO] [1737250110.215827]: reset action history +[INFO] [1737250113.355429]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_18_2025-01-18-20-28-33.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:33279/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-55]: started with pid [2572459] +process[print_text-56]: started with pid [2572460] +process[rosbag_record-57]: started with pid [2572461] +started bagging! +[INFO] [1737250147.832898]: Storing episode... +[print_text-56] killing on exit +[rosbag_record-57] killing on exit +[pub_text-55] killing on exit +[INFO] [1737250148.741336]: Stored episode 19. +[INFO] [1737250148.788457]: Saving video... +[INFO] [1737250149.301728]: Video saved! +Starting episode 19 +[INFO] [1737250149.519034]: resetting environment +[INFO] [1737250149.563649]: cleared current plan +[INFO] [1737250149.569059]: moving to home +[WARN] [1737250149.172996]: Plan exhausted +[WARN] [1737250149.197495]: Plan exhausted +[WARN] [1737250149.237433]: Plan exhausted +[WARN] [1737250149.274200]: Plan exhausted +[WARN] [1737250149.321313]: Plan exhausted +[WARN] [1737250149.354652]: Plan exhausted +[WARN] [1737250149.395664]: Plan exhausted +[WARN] [1737250149.458762]: Plan exhausted +[WARN] [1737250149.536460]: Plan exhausted +[WARN] [1737250149.550680]: Plan exhausted +[WARN] [1737250149.554357]: Plan exhausted +[INFO] [1737250152.990805]: reached home +[INFO] [1737250153.014844]: reset action history +[INFO] [1737250156.188280]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737250196.922947]: resetting environment +[INFO] [1737250197.012977]: cleared current plan +[INFO] [1737250197.030416]: moving to home +[INFO] [1737250198.041996]: reached home +[INFO] [1737250198.083637]: reset action history +[INFO] [1737250201.473890]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/episode_rosbags/episode_19_2025-01-18-20-30-01.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2569579.log +started roslaunch server http://158.130.50.37:40689/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-58]: started with pid [2572679] +process[print_text-59]: started with pid [2572680] +process[rosbag_record-60]: started with pid [2572681] +started bagging! +For topic gripper_width: timestamp difference is 104412909 for nearest: 1737250230668919201 - target: 1737250230564506292 at idx 249 +For topic gripper_width: timestamp difference is 103690622 for nearest: 1737250230868869541 - target: 1737250230765178919 at idx 249 +For topic gripper_width: timestamp difference is 101066671 for nearest: 1737250234268915258 - target: 1737250234167848587 at idx 249 +For topic gripper_width: timestamp difference is 101255090 for nearest: 1737250234468942792 - target: 1737250234367687702 at idx 249 +For topic gripper_width: timestamp difference is 103125509 for nearest: 1737250236868917402 - target: 1737250236765791893 at idx 249 +[INFO] [1737250239.463866]: Storing episode... +[print_text-59] killing on exit[rosbag_record-60] killing on exit +[pub_text-58] killing on exit +[INFO] [1737250240.306415]: Stored episode 20. +[INFO] [1737250240.324157]: Saving video... +[INFO] [1737250241.060668]: Video saved! +[INFO] [1737250241.194716]: resetting environment +[INFO] [1737250241.257884]: cleared current plan +[INFO] [1737250241.272096]: moving to home +[WARN] [1737250240.846257]: Plan exhausted +[WARN] [1737250240.875108]: Plan exhausted +[WARN] [1737250240.914510]: Plan exhausted +[WARN] [1737250240.957426]: Plan exhausted +[WARN] [1737250241.017948]: Plan exhausted +[WARN] [1737250241.068446]: Plan exhausted +[WARN] [1737250241.117911]: Plan exhausted +[WARN] [1737250241.150340]: Plan exhausted +[WARN] [1737250241.179952]: Plan exhausted +[WARN] [1737250241.205984]: Plan exhausted +[WARN] [1737250241.233999]: Plan exhausted +[INFO] [1737250244.888374]: reached home +[INFO] [1737250244.902541]: reset action history +[INFO] [1737250248.188859]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/requirements.txt b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch new file mode 100644 index 0000000000000000000000000000000000000000..a5389e74874ef2add6aa3eadc61bc8ee3ea4ebde --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch @@ -0,0 +1,119 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..03918f5 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 5 ++num_eval: 20 + + debug_timestamps: False + open_loop: False +@@ -132,15 +132,25 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1000_0' # all + + # all books, 10/20 demos per book ++# crp_D+act history ++# wandb_run_id: '3456_0' ++# wandb_run_id: '3456_1' # seed 1 ++# wandb_run_id: '3456_2' # seed 2 ++ ++# all books, 06/20 demos per book ++# crp_D+act history ++wandb_run_id: '3465_0' ++# wandb_run_id: '3465_1' # seed 1 ++# wandb_run_id: '3465_2' # seed 2 ++ + # RGBD+mask+act history +-# wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history +@@ -191,7 +201,7 @@ wandb_run_id: '1017_0' # dataset shuffle seed 1 + true_action_history: false + wandb_notes: null + +-checkpoint_epoch: 12000 # for depth + all contact ++checkpoint_epoch: 7500 # for depth + all contact + load_residual_weight: false + + checkpoint_root_dir: '/home/${oc.env:USER}/fish_leon/FISH' +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..9509617 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,15 +167,21 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ # '3456_0', ++ # '3456_1', ++ # '3456_2', ++ '3465_0', ++ '3465_1', ++ '3465_2', + ] + +-checkpoint_epoch = 12000 ++checkpoint_epoch = 7500 + + # use subprocess to download the model checkpoints in parallel + process_list = [] +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-metadata.json b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..3e2baf9fdd6ca0ece788d76bdd57595c13f8b60d --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-19T01:14:03.646859", + "startedAt": "2025-01-19T01:14:03.251322", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "ec42957dc10d08dbd8b134558fcd8f9e470ccc31" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3561.5105, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4492.066, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.032, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.021, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 605.3610420227051 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..888f960f6813d1b2fa7aca65dd74213a79868ac4 --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "958873708057cb5c96d07eec36f6c0e68759bdf4c854f4fae4331f95dc510299", "size": 1142998, "path": "media/videos/eval/0_eval_0_958873708057cb5c96d0.mp4"}, "global_step": 2659, "_timestamp": 1737250252.7608755, "_runtime": 1009.493985414505, "_step": 41, "eval/1_eval": {"_type": "video-file", "sha256": "c93c245f94bccf988bf81c13b34f2b4f94a31c61b0b700a37cd4e15dacb444df", "size": 1151340, "path": "media/videos/eval/1_eval_1_c93c245f94bccf988bf8.mp4"}, "eval/num_success": 18.0, "episode": 40.0, "eval/success_rate": 0.8999999761581421, "eval/success_rate_lb_0.01": 0.6478986144065857, "eval/success_rate_ub_0.01": 0.9880934953689575, "eval/success_rate_lb_0.025": 0.6879684925079346, "eval/success_rate_ub_0.025": 0.974714457988739, "eval/success_rate_lb_0.05": 0.7650595307350159, "eval/success_rate_ub_0.05": 0.9736677408218384, "eval/success_rate_lb_0.1": 0.7752392292022705, "eval/success_rate_ub_0.1": 0.9721130728721619, "eval/2_eval": {"_type": "video-file", "sha256": "35d91a735aea17ce0770b204540d9606ca19ba47596be1fa84c03e8a76d48b12", "size": 1160864, "path": "media/videos/eval/2_eval_2_35d91a735aea17ce0770.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "c4ac9168429d53521dfd398dfdf30167e676a6d017c37e6719de8c077ae2d6c5", "size": 1157641, "path": "media/videos/eval/3_eval_3_c4ac9168429d53521dfd.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "73cfbcf4697dd02a170455072cbbb3a1ca0352372cb4a0797e92ead11895af34", "size": 1154605, "path": "media/videos/eval/4_eval_4_73cfbcf4697dd02a1704.mp4"}, "eval/5_eval": {"_type": "video-file", "sha256": "8eee7b4209c8cc001b291cef3b235d9ad935ca7d79ef91da31d83f4b2565d510", "size": 1146010, "path": "media/videos/eval/5_eval_5_8eee7b4209c8cc001b29.mp4"}, "eval/6_eval": {"_type": "video-file", "sha256": "6f92e10ddc5700f95864b3bd37bb543a48502145e8956f152881dac10ecc3e27", "size": 1142252, "path": "media/videos/eval/6_eval_6_6f92e10ddc5700f95864.mp4"}, "eval/7_eval": {"_type": "video-file", "sha256": "ec916f9f1cb86bb268d4b76b9be7a6f2b0e4f3512ce092e5d968e616457e41ba", "size": 1230184, "path": "media/videos/eval/7_eval_7_ec916f9f1cb86bb268d4.mp4"}, "eval/8_eval": {"_type": "video-file", "sha256": "c1b631f94f507d0b802907ef582d02c763dd6faa3b1b3b36bfa5e09e9ab836bc", "size": 1261905, "path": "media/videos/eval/8_eval_8_c1b631f94f507d0b8029.mp4"}, "eval/9_eval": {"_type": "video-file", "sha256": "45f46258144727274eca27c01c0f84d776e772cc618e870d73101f7b953894fb", "size": 1162867, "path": "media/videos/eval/9_eval_9_45f46258144727274eca.mp4"}, "eval/10_eval": {"_type": "video-file", "sha256": "dbfaa8aaf0c838051064d1f52334d6d83d61935c556bc18851fb329b8e51a97b", "size": 1290176, "path": "media/videos/eval/10_eval_10_dbfaa8aaf0c838051064.mp4"}, "eval/11_eval": {"_type": "video-file", "sha256": "359c87ada34d682bae4526c67a0e1660ebc14229bbfea9ebda9878bc7cbe49e2", "size": 1165825, "path": "media/videos/eval/11_eval_11_359c87ada34d682bae45.mp4"}, "eval/12_eval": {"_type": "video-file", "sha256": "50069bff80de5f04623040da25c9abba38c19c8c6f485f7ecb364ef9c652a551", "size": 1365675, "path": "media/videos/eval/12_eval_12_50069bff80de5f046230.mp4"}, "eval/13_eval": {"_type": "video-file", "sha256": "114ad2ddaaeca4cacdd5ae91c8134dd40a3d5b8d16f5c1538175afff73d03d34", "size": 1346252, "path": "media/videos/eval/13_eval_13_114ad2ddaaeca4cacdd5.mp4"}, "eval/14_eval": {"_type": "video-file", "sha256": "d43b1e73114608f44587bfbbd894e999f6f34aa1b616cbab8d4f16ce3f271adc", "size": 1413320, "path": "media/videos/eval/14_eval_14_d43b1e73114608f44587.mp4"}, "eval/15_eval": {"_type": "video-file", "sha256": "a134ab49305f64bfef917aadcc5002acbf1b6c847787e9add516af98661440cb", "size": 1461649, "path": "media/videos/eval/15_eval_15_a134ab49305f64bfef91.mp4"}, "eval/16_eval": {"_type": "video-file", "sha256": "fba70f3485b0b97a5e738e162bd29deed4662a79fc5279012753333773ee3b10", "size": 1472547, "path": "media/videos/eval/16_eval_16_fba70f3485b0b97a5e73.mp4"}, "eval/17_eval": {"_type": "video-file", "sha256": "70585fb0ca24437a0f9f8d8ce89db96ec7d90d0e2abf545ad2ba5f0a1b9d31c9", "size": 1504233, "path": "media/videos/eval/17_eval_17_70585fb0ca24437a0f9f.mp4"}, "eval/18_eval": {"_type": "video-file", "sha256": "666566ff2e12fdecef308d665951ca56d112b39565b2350089304d70dea6847b", "size": 1399960, "path": "media/videos/eval/18_eval_18_666566ff2e12fdecef30.mp4"}, "eval/19_eval": {"_type": "video-file", "sha256": "d8301c722455e230f538b18357aed48ab0500a9465cfbf6fbf0edd142fcf14c5", "size": 1567147, "path": "media/videos/eval/19_eval_19_d8301c722455e230f538.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "54e2754b6d7019322e440221b7b11780c96cc3143c38f4fe7aace7d8c88c52a6", "size": 398, "artifact_path": "wandb-client-artifact://11yiacj29zq312vh3js1vpv4rf0473265x9pkfucev37a68gnchjt7gu5gntpydmabhgrpo63cx57sz7prntnxxtqrd3plx2l7601nifwdyr6oo1tg2s8k1thnw88p15/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://s2u50swvghoqean2e4m9iix4vdmjtaha8y452khy73ro2l0cdb5bdaoe5gmw9lz9x5eme3po5692ccd7w3ns7w8jwfgykedjyap950f2x5t4l0kwv8iax5gdpyo1o4er:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_20_54e2754b6d7019322e44.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 39, "total_num_episodes": 40, "total_success_rate": 0.975, "total_success_rate_lb_0.01": 0.8683311640479618, "total_success_rate_ub_0.01": 0.9995417957126389, "total_success_rate_lb_0.025": 0.8927702902962017, "total_success_rate_ub_0.025": 0.9965071956565379, "total_success_rate_lb_0.05": 0.9260990469795616, "total_success_rate_ub_0.05": 0.9941391329224076, "total_success_rate_lb_0.1": 0.9064763336560784, "total_success_rate_ub_0.1": 0.9959524566247081, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "84d6d84980b416ef739f61e901c0ed4e8f18c17637701aeb0ecf3f638498a023", "size": 507, "artifact_path": "wandb-client-artifact://elrxjw15tndq9tp5tuzp4zjsl1hxkqzkgq45hs1bjl8h22b6f9fo53e1cc0on5kx6fbmbuxapo480vxt6oypgijreuy4ylogklchyyn5z2l51b2haiv9xd1dcp9h78cv/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://7dggooij5jzcvufaceatcxtgnv92zub41ghpiymc09aivsbxmforw1nqhhswhc3tq451wq43kagap5yyzn9o1a2gm2aiiq0js2606wqyf6fmfal0309ma713lk1067ws:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_21_84d6d84980b416ef739f.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 1031}, "eval/total_success_rate": 0.9750000238418579, "eval/total_success_rate_lb_0.01": 0.8683311343193054, "eval/total_success_rate_ub_0.01": 0.9995418190956116, "eval/total_success_rate_lb_0.025": 0.8927702903747559, "eval/total_success_rate_ub_0.025": 0.9965071678161621, "eval/total_success_rate_lb_0.05": 0.9260990619659424, "eval/total_success_rate_ub_0.05": 0.9941391348838806, "eval/total_success_rate_lb_0.1": 0.9064763188362122, "eval/total_success_rate_ub_0.1": 0.9959524273872375} \ No newline at end of file diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/logs/debug-internal.log b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d938868f7208fce03fe29a0b209acee1e9f1ed1b --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/logs/debug-internal.log @@ -0,0 +1,2207 @@ +2025-01-18 20:14:03,266 INFO StreamThr :2569743 [internal.py:wandb_internal():85] W&B internal server running at pid: 2569743, started at: 2025-01-18 20:14:03.266253 +2025-01-18 20:14:03,267 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status +2025-01-18 20:14:03,279 INFO WriterThread:2569743 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/run-h3eacbh9.wandb +2025-01-18 20:14:03,279 DEBUG SenderThread:2569743 [sender.py:send():379] send: header +2025-01-18 20:14:03,286 DEBUG SenderThread:2569743 [sender.py:send():379] send: run +2025-01-18 20:14:03,527 INFO SenderThread:2569743 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files +2025-01-18 20:14:03,527 INFO SenderThread:2569743 [sender.py:_start_run_threads():1188] run started: h3eacbh9 with start time 1737249243.26689 +2025-01-18 20:14:03,542 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: check_version +2025-01-18 20:14:03,542 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: check_version +2025-01-18 20:14:03,621 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: run_start +2025-01-18 20:14:03,633 DEBUG HandlerThread:2569743 [system_info.py:__init__():26] System info init +2025-01-18 20:14:03,633 DEBUG HandlerThread:2569743 [system_info.py:__init__():41] System info init done +2025-01-18 20:14:03,633 INFO HandlerThread:2569743 [system_monitor.py:start():194] Starting system monitor +2025-01-18 20:14:03,633 INFO SystemMonitor:2569743 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-18 20:14:03,633 INFO HandlerThread:2569743 [system_monitor.py:probe():214] Collecting system info +2025-01-18 20:14:03,634 INFO SystemMonitor:2569743 [interfaces.py:start():188] Started cpu monitoring +2025-01-18 20:14:03,635 INFO SystemMonitor:2569743 [interfaces.py:start():188] Started disk monitoring +2025-01-18 20:14:03,635 INFO SystemMonitor:2569743 [interfaces.py:start():188] Started gpu monitoring +2025-01-18 20:14:03,636 INFO SystemMonitor:2569743 [interfaces.py:start():188] Started memory monitoring +2025-01-18 20:14:03,636 INFO SystemMonitor:2569743 [interfaces.py:start():188] Started network monitoring +2025-01-18 20:14:03,646 DEBUG HandlerThread:2569743 [system_info.py:probe():152] Probing system +2025-01-18 20:14:03,648 DEBUG HandlerThread:2569743 [system_info.py:_probe_git():137] Probing git +2025-01-18 20:14:03,652 DEBUG HandlerThread:2569743 [system_info.py:_probe_git():145] Probing git done +2025-01-18 20:14:03,652 DEBUG HandlerThread:2569743 [system_info.py:probe():200] Probing system done +2025-01-18 20:14:03,652 DEBUG HandlerThread:2569743 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-19T01:14:03.646859', 'startedAt': '2025-01-19T01:14:03.251322', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.5105, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4492.066, 'min': 800.0, 'max': 4700.0}, {'current': 4500.032, 'min': 800.0, 'max': 4700.0}, {'current': 4500.021, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 605.3610420227051}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-18 20:14:03,653 INFO HandlerThread:2569743 [system_monitor.py:probe():224] Finished collecting system info +2025-01-18 20:14:03,653 INFO HandlerThread:2569743 [system_monitor.py:probe():227] Publishing system info +2025-01-18 20:14:03,653 DEBUG HandlerThread:2569743 [system_info.py:_save_code():44] Saving code +2025-01-18 20:14:03,658 DEBUG HandlerThread:2569743 [system_info.py:_save_code():67] Saving code done +2025-01-18 20:14:03,658 DEBUG HandlerThread:2569743 [system_info.py:_save_patches():84] Saving git patches +2025-01-18 20:14:03,753 DEBUG HandlerThread:2569743 [system_info.py:_save_patches():126] Saving git patches done +2025-01-18 20:14:03,753 INFO HandlerThread:2569743 [system_monitor.py:probe():229] Finished publishing system info +2025-01-18 20:14:03,757 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:14:03,757 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-18 20:14:03,757 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-18 20:14:03,757 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-18 20:14:03,757 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-18 20:14:03,984 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: python_packages +2025-01-18 20:14:03,985 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: python_packages +2025-01-18 20:14:03,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:14:03,985 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:14:03,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:03,987 INFO HandlerThread:2569743 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/tb" + save: true +} +control { + relay_id: "177c0a6323c5493290332c4877990993" +} +_info { + stream_id: "h3eacbh9" +} + +2025-01-18 20:14:03,988 DEBUG HandlerThread:2569743 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-18 20:14:04,032 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/y2s8i8aq-wandb-metadata.json +2025-01-18 20:14:04,035 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/e8ijb1ii-code/FISH/eval_robot.py +2025-01-18 20:14:04,052 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/wdl6mw9j-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 20:14:04,053 DEBUG SenderThread:2569743 [sender.py:send():379] send: telemetry +2025-01-18 20:14:04,054 DEBUG SenderThread:2569743 [sender.py:send():379] send: tbrecord +2025-01-18 20:14:04,054 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:14:04,054 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 with policy live +2025-01-18 20:14:04,060 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/8mksntvi-diff.patch +2025-01-18 20:14:04,246 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/yyc7lpp2-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:14:04,531 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/requirements.txt +2025-01-18 20:14:04,531 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 20:14:04,531 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/diff.patch +2025-01-18 20:14:04,532 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:14:04,532 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-metadata.json +2025-01-18 20:14:04,532 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/code/FISH/eval_robot.py +2025-01-18 20:14:04,533 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:04,533 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/code/FISH +2025-01-18 20:14:04,533 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/code +2025-01-18 20:14:04,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:05,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:06,528 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:06,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:07,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:08,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:08,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:09,078 DEBUG SenderThread:2569743 [sender.py:send():379] send: config +2025-01-18 20:14:09,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:10,529 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:10,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:11,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:12,530 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:12,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:13,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:13,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:14,531 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:14,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:15,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:16,531 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:16,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:17,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:18,532 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:18,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:14:18,985 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:14:19,029 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:19,048 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:19,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:20,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:21,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:22,533 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:22,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:23,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:24,155 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:24,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:25,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:26,534 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:26,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:27,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:28,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:29,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:29,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:30,989 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:31,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:32,989 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:33,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:14:33,985 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:14:34,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:34,989 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:35,067 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:35,537 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/config.yaml +2025-01-18 20:14:35,989 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:36,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:37,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:38,989 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:39,989 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:40,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:40,989 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:41,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:42,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:43,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:44,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:45,991 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:46,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:46,991 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:47,991 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:48,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:14:48,985 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:14:48,991 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:49,991 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:50,540 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:50,991 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:51,916 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:14:51,917 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:14:51,917 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_958873708057cb5c96d0.mp4 with policy now +2025-01-18 20:14:51,918 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:14:51,918 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:14:51,918 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:14:51,991 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:52,192 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/7e8whsay-media/videos/eval/0_eval_0_958873708057cb5c96d0.mp4 +2025-01-18 20:14:52,541 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:52,541 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:14:52,542 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/0_eval_0_958873708057cb5c96d0.mp4 +2025-01-18 20:14:52,542 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:14:52,542 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos +2025-01-18 20:14:52,542 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media +2025-01-18 20:14:52,848 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/sufv42hu-media/videos/eval/0_eval_0_958873708057cb5c96d0.mp4 +2025-01-18 20:14:52,924 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:52,992 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:53,992 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:54,542 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:54,992 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:55,992 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:56,543 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:14:56,992 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:57,993 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:58,236 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:14:58,993 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:14:59,993 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:00,544 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:00,993 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:01,993 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:02,993 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:03,636 DEBUG SystemMonitor:2569743 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-18 20:15:03,637 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:15:03,637 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:03,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:15:03,986 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:15:04,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:04,993 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:05,993 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:06,993 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:07,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:08,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:08,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:09,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:10,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:11,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:12,548 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:15:12,806 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/3xm431t2-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:15:12,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:13,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:13,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:14,549 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:14,995 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:15,995 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:16,995 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:17,995 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:18,550 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:18,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:15:18,986 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:15:19,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:19,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:19,995 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:20,995 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:21,996 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:22,996 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:23,996 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:24,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:24,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:25,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:26,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:27,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:28,554 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:28,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:29,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:29,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:30,554 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:30,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:31,998 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:32,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:33,637 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:15:33,985 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:15:33,986 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:15:34,029 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:34,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:35,124 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:35,998 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:36,998 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:37,998 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:38,998 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:39,998 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:40,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:40,998 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:41,998 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:42,998 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:43,999 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:44,505 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:15:44,505 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_c93c245f94bccf988bf8.mp4 with policy now +2025-01-18 20:15:44,506 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:15:44,506 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:15:44,506 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:15:44,507 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:15:44,559 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:15:44,559 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:44,559 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/1_eval_1_c93c245f94bccf988bf8.mp4 +2025-01-18 20:15:44,559 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:15:44,767 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/b6ysw7lj-media/videos/eval/1_eval_1_c93c245f94bccf988bf8.mp4 +2025-01-18 20:15:44,999 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:45,028 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/n8yvhgw8-media/videos/eval/1_eval_1_c93c245f94bccf988bf8.mp4 +2025-01-18 20:15:46,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:46,518 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:46,559 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:47,002 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:48,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:48,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:15:48,994 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:15:49,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:50,002 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:50,561 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:51,002 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:51,953 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:52,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:52,562 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:53,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:54,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:55,006 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:56,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:56,563 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:15:57,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:57,218 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:15:57,391 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:15:57,391 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:15:57,392 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:15:57,563 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:15:58,004 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:15:58,564 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:15:59,004 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:00,004 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:01,004 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:02,004 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:02,333 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:02,565 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:03,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:03,638 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:16:03,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:16:03,986 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:16:04,029 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:05,004 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:06,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:07,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:08,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:08,075 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:09,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:10,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:11,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:12,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:13,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:13,906 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:14,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:15,006 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:16,006 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:16,570 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:17,006 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:18,006 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:18,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:16:18,986 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:16:19,029 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:19,082 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:20,006 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:21,006 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:22,007 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:23,007 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:24,007 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:24,992 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:25,007 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:26,007 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:27,007 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:28,008 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:28,201 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:16:28,201 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:16:28,202 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_35d91a735aea17ce0770.mp4 with policy now +2025-01-18 20:16:28,202 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:16:28,203 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:16:28,203 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:16:28,506 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/p1hyhc5j-media/videos/eval/2_eval_2_35d91a735aea17ce0770.mp4 +2025-01-18 20:16:28,574 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:16:28,574 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:28,574 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/2_eval_2_35d91a735aea17ce0770.mp4 +2025-01-18 20:16:28,574 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:16:28,848 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/nkyldyvs-media/videos/eval/2_eval_2_35d91a735aea17ce0770.mp4 +2025-01-18 20:16:29,009 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:29,999 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:30,009 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:30,574 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:31,011 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:32,012 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:33,014 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:33,639 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:16:33,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:16:33,991 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:16:34,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:34,576 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:35,009 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:35,661 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:36,009 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:36,577 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:37,014 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:38,011 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:39,014 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:40,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:40,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:41,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:42,013 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:42,579 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:16:42,832 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/ccky18u0-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:16:43,013 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:43,234 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:16:43,235 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:16:43,235 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:16:43,579 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:16:44,020 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:44,579 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:45,016 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:45,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:46,014 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:47,014 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:48,014 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:48,581 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:48,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:16:48,986 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:16:49,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:50,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:51,014 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:51,072 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:52,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:52,582 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:53,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:54,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:54,583 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:16:55,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:56,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:56,996 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:16:57,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:58,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:16:59,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:00,016 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:01,016 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:02,016 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:02,996 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:03,017 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:03,639 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:17:03,986 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:17:03,987 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:17:04,029 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:05,016 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:06,017 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:07,017 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:08,017 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:08,077 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:09,017 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:10,017 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:11,017 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:12,017 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:13,018 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:13,243 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:14,028 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:14,459 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:17:14,460 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c4ac9168429d53521dfd.mp4 with policy now +2025-01-18 20:17:14,462 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:17:14,463 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:17:14,463 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:17:14,463 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:17:14,590 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:17:14,590 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:17:14,590 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/3_eval_3_c4ac9168429d53521dfd.mp4 +2025-01-18 20:17:14,590 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:17:14,784 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/gfaw09av-media/videos/eval/3_eval_3_c4ac9168429d53521dfd.mp4 +2025-01-18 20:17:15,034 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:15,059 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/bra3uouf-media/videos/eval/3_eval_3_c4ac9168429d53521dfd.mp4 +2025-01-18 20:17:16,030 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:16,590 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:17:17,025 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:18,036 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:19,000 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:17:19,000 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:17:19,032 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:19,159 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:20,029 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:20,592 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:17:21,027 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:22,035 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:22,593 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:17:23,030 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:24,031 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:25,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:25,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:26,034 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:27,031 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:27,289 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:17:27,290 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:17:27,290 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:17:27,594 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:17:27,594 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:17:28,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:28,595 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:17:29,040 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:30,005 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:30,035 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:30,595 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:17:31,034 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:32,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:32,596 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:17:33,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:33,640 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:17:33,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:17:33,987 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:17:34,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:34,597 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:17:35,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:35,569 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:36,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:36,598 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:17:37,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:38,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:39,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:40,034 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:41,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:41,034 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:42,034 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:43,034 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:44,035 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:45,035 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:46,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:46,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:47,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:48,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:48,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:17:48,987 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:17:49,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:50,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:51,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:51,106 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:52,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:53,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:54,038 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:55,038 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:56,038 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:57,003 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:17:57,038 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:58,038 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:59,055 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:17:59,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:17:59,165 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:17:59,165 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_73cfbcf4697dd02a1704.mp4 with policy now +2025-01-18 20:17:59,166 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:17:59,166 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:17:59,167 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:17:59,424 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/nffdq5oc-media/videos/eval/4_eval_4_73cfbcf4697dd02a1704.mp4 +2025-01-18 20:17:59,606 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:17:59,606 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/4_eval_4_73cfbcf4697dd02a1704.mp4 +2025-01-18 20:17:59,606 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:17:59,877 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/8t0wjxsc-media/videos/eval/4_eval_4_73cfbcf4697dd02a1704.mp4 +2025-01-18 20:18:00,048 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:00,606 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:01,060 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:02,065 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:02,178 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:03,055 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:03,646 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:18:03,996 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:18:03,997 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:18:04,056 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:04,608 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:05,053 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:06,049 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:06,608 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:07,052 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:07,630 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:08,057 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:09,055 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:10,056 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:10,345 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:18:10,345 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:18:10,345 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:18:10,610 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:18:10,610 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:18:10,610 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:10,864 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/16d0k2cs-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:18:11,060 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:12,057 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:12,610 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:12,969 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:13,068 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:14,057 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:14,611 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:15,057 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:16,057 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:16,612 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:17,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:18,004 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:18,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:18,987 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:18:18,988 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:18:19,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:20,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:21,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:22,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:23,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:23,140 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:24,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:25,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:26,059 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:27,059 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:28,059 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:29,004 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:29,060 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:30,060 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:31,060 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:32,060 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:32,618 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:33,061 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:33,641 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:18:33,988 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:18:33,988 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:18:34,061 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:34,149 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:35,060 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:36,061 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:37,061 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:38,061 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:39,061 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:40,004 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:40,061 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:41,065 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:41,906 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:18:41,906 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_8eee7b4209c8cc001b29.mp4 with policy now +2025-01-18 20:18:41,909 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:18:41,909 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:18:41,910 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:18:41,911 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:18:42,065 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:42,166 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/tyzl746m-media/videos/eval/5_eval_5_8eee7b4209c8cc001b29.mp4 +2025-01-18 20:18:42,621 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:18:42,622 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:42,622 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/5_eval_5_8eee7b4209c8cc001b29.mp4 +2025-01-18 20:18:42,622 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:18:42,901 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/i811lut9-media/videos/eval/5_eval_5_8eee7b4209c8cc001b29.mp4 +2025-01-18 20:18:43,065 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:43,622 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:44,077 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:44,623 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:45,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:45,068 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:46,089 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:46,623 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:47,074 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:48,079 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:48,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:18:48,990 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:18:49,077 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:50,081 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:50,096 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:50,625 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:51,075 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:52,076 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:53,073 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:54,082 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:54,400 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:18:54,400 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:18:54,401 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:18:54,627 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:18:54,627 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:18:55,087 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:55,197 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:18:56,081 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:56,628 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:18:57,083 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:58,097 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:18:59,080 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:00,080 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:00,629 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:01,018 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:01,080 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:02,081 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:03,081 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:03,641 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:19:03,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:19:03,990 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:19:04,081 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:05,081 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:06,081 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:06,086 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:07,082 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:08,081 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:09,082 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:10,083 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:11,083 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:12,018 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:12,083 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:13,083 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:14,083 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:15,083 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:16,083 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:17,018 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:17,083 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:18,084 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:18,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:19:18,990 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:19:19,084 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:20,084 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:21,084 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:22,084 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:22,144 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:23,084 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:24,084 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:25,084 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:26,088 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:26,233 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:19:26,233 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_6f92e10ddc5700f95864.mp4 with policy now +2025-01-18 20:19:26,244 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:19:26,245 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:19:26,245 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:19:26,246 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:19:26,493 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/85f3h9co-media/videos/eval/6_eval_6_6f92e10ddc5700f95864.mp4 +2025-01-18 20:19:26,639 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:19:26,639 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:26,639 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/6_eval_6_6f92e10ddc5700f95864.mp4 +2025-01-18 20:19:26,639 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:19:26,921 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/kaay7jpj-media/videos/eval/6_eval_6_6f92e10ddc5700f95864.mp4 +2025-01-18 20:19:27,093 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:27,269 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:28,094 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:28,640 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:29,093 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:30,095 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:30,640 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:31,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:32,116 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:32,737 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:33,098 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:33,641 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:19:33,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:19:33,994 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:19:34,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:34,642 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:35,114 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:36,115 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:37,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:37,778 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:38,114 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:38,455 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:19:38,455 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:19:38,456 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:19:38,644 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:19:38,644 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:19:38,644 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:38,895 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/irpocnqm-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:19:39,118 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:40,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:40,645 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:41,118 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:42,116 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:42,645 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:43,122 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:43,678 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:44,116 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:44,647 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:45,116 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:46,116 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:46,647 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:19:47,116 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:48,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:48,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:19:48,990 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:19:49,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:49,145 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:50,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:51,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:52,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:53,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:54,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:55,024 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:19:55,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:56,118 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:57,118 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:58,118 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:19:59,118 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:00,024 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:00,118 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:01,118 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:02,118 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:03,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:03,642 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:20:03,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:20:03,990 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:20:04,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:05,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:05,152 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:06,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:07,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:08,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:09,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:10,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:10,725 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:11,141 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:12,084 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:20:12,084 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_ec916f9f1cb86bb268d4.mp4 with policy now +2025-01-18 20:20:12,091 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:20:12,092 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:20:12,092 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:20:12,092 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:20:12,129 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:12,350 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/zrcdmlmp-media/videos/eval/7_eval_7_ec916f9f1cb86bb268d4.mp4 +2025-01-18 20:20:12,658 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:20:12,658 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:12,658 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/7_eval_7_ec916f9f1cb86bb268d4.mp4 +2025-01-18 20:20:12,658 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:20:12,928 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/90kl4c8p-media/videos/eval/7_eval_7_ec916f9f1cb86bb268d4.mp4 +2025-01-18 20:20:13,148 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:13,659 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:14,141 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:14,659 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:15,139 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:15,752 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:16,156 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:16,660 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:17,131 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:18,148 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:18,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:20:18,991 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:20:19,155 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:20,150 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:20,508 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:20:20,508 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:20:20,522 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:20:20,661 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:20:20,662 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:20:20,662 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:20,861 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:21,176 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:22,159 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:22,662 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:23,176 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:24,161 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:24,663 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:25,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:26,034 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:26,174 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:27,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:28,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:28,665 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:29,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:30,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:30,665 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:31,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:32,029 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:32,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:33,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:33,642 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:20:33,990 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:20:33,991 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:20:34,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:35,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:36,164 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:37,127 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:37,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:38,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:39,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:40,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:41,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:42,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:43,030 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:43,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:44,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:45,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:46,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:47,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:48,030 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:48,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:48,991 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:20:48,991 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:20:49,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:50,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:51,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:52,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:53,108 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:53,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:54,180 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:54,674 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:54,929 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:20:54,929 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_c1b631f94f507d0b8029.mp4 with policy now +2025-01-18 20:20:54,943 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:20:54,944 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:20:54,944 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:20:54,944 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:20:55,172 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:55,190 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/mjyndchd-media/videos/eval/8_eval_8_c1b631f94f507d0b8029.mp4 +2025-01-18 20:20:55,675 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:20:55,675 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/8_eval_8_c1b631f94f507d0b8029.mp4 +2025-01-18 20:20:55,675 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:20:55,978 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/alo1tma0-media/videos/eval/8_eval_8_c1b631f94f507d0b8029.mp4 +2025-01-18 20:20:56,184 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:56,675 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:57,186 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:57,676 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:20:58,191 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:20:58,604 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:20:59,200 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:00,192 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:00,677 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:01,210 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:02,217 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:02,678 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:03,212 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:03,643 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:21:03,644 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:04,013 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:21:04,014 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:21:04,214 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:04,560 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:21:04,561 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:21:04,561 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:21:04,678 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:21:04,679 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:21:04,679 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:04,937 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/r2bwdo23-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:21:05,216 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:06,220 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:06,679 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:07,216 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:08,207 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:08,680 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:09,040 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:09,206 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:10,206 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:10,681 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:11,206 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:12,206 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:13,206 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:14,040 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:14,207 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:15,207 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:16,207 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:17,207 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:18,208 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:18,992 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:21:18,992 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:21:19,146 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:19,208 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:20,208 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:21,208 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:22,208 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:23,208 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:24,208 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:25,040 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:25,209 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:26,209 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:27,209 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:28,209 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:29,210 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:30,040 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:30,210 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:31,210 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:32,210 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:33,210 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:33,643 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:21:33,992 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:21:33,993 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:21:34,210 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:35,218 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:35,612 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:36,221 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:36,387 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:21:36,387 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_45f46258144727274eca.mp4 with policy now +2025-01-18 20:21:36,389 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:21:36,390 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:21:36,390 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:21:36,390 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:21:36,644 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/cylhyune-media/videos/eval/9_eval_9_45f46258144727274eca.mp4 +2025-01-18 20:21:36,692 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:21:36,692 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:36,692 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/9_eval_9_45f46258144727274eca.mp4 +2025-01-18 20:21:36,692 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:21:36,966 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/8vkplk5e-media/videos/eval/9_eval_9_45f46258144727274eca.mp4 +2025-01-18 20:21:37,219 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:37,692 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:38,237 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:38,692 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:39,238 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:39,693 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:40,241 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:40,693 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:41,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:41,245 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:42,238 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:42,694 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:43,225 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:44,237 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:44,695 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:45,246 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:46,238 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:46,351 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:46,612 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:21:46,613 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:21:46,613 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:21:46,696 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:21:46,696 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:21:47,248 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:48,244 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:48,696 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:49,019 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:21:49,019 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:21:49,242 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:50,258 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:51,252 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:51,473 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:52,251 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:52,698 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:53,275 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:54,267 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:55,257 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:56,257 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:56,699 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:21:57,045 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:21:57,257 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:58,257 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:21:59,257 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:00,258 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:01,258 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:02,059 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:02,258 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:03,258 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:03,644 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:22:03,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:22:03,998 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:22:04,258 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:05,259 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:06,259 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:07,156 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:07,259 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:08,259 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:09,259 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:10,259 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:11,259 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:12,259 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:12,706 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:12,841 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:13,260 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:14,260 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:15,260 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:16,260 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:16,708 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:17,260 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:18,242 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:18,260 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:18,997 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:22:18,998 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:22:19,261 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:20,260 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:21,261 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:22,262 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:22,485 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:22:22,485 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_dbfaa8aaf0c838051064.mp4 with policy now +2025-01-18 20:22:22,487 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:22:22,488 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:22:22,488 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:22:22,489 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:22:22,710 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:22:22,710 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:22,710 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/10_eval_10_dbfaa8aaf0c838051064.mp4 +2025-01-18 20:22:22,710 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:22:22,739 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/d4kkewji-media/videos/eval/10_eval_10_dbfaa8aaf0c838051064.mp4 +2025-01-18 20:22:23,033 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/434t8aip-media/videos/eval/10_eval_10_dbfaa8aaf0c838051064.mp4 +2025-01-18 20:22:23,281 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:23,548 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:24,268 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:24,711 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:25,284 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:25,711 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:26,292 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:27,293 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:28,310 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:28,712 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:29,077 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:29,287 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:30,300 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:30,713 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:31,315 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:32,302 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:33,320 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:33,644 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:22:34,024 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:22:34,024 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:22:34,168 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:34,317 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:35,310 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:36,323 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:37,338 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:38,329 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:39,327 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:40,096 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:40,332 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:41,336 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:42,330 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:43,337 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:44,337 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:45,364 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:46,095 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:46,333 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:47,344 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:48,341 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:48,686 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:22:48,686 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:22:48,686 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:22:48,720 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:22:48,720 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:22:48,720 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:48,993 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/8xpe97fa-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:22:49,017 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:22:49,018 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:22:49,354 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:50,359 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:50,721 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:51,264 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:51,360 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:52,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:53,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:54,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:54,722 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:22:55,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:56,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:57,075 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:22:57,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:58,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:22:59,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:00,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:01,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:02,075 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:02,356 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:03,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:03,645 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:23:04,001 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:23:04,002 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:23:04,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:05,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:06,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:07,165 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:07,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:08,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:09,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:10,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:11,357 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:12,358 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:13,076 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:13,358 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:14,358 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:14,730 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:15,358 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:16,358 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:17,358 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:18,076 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:18,358 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:19,002 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:23:19,002 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:23:19,359 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:20,360 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:20,733 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:20,733 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/11_eval_11_359c87ada34d682bae45.mp4 +2025-01-18 20:23:20,733 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:23:20,742 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:23:20,742 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_359c87ada34d682bae45.mp4 with policy now +2025-01-18 20:23:20,744 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:23:20,744 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:23:20,745 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:23:20,752 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:23:20,995 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/4vb69597-media/videos/eval/11_eval_11_359c87ada34d682bae45.mp4 +2025-01-18 20:23:21,251 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/li0z0nfk-media/videos/eval/11_eval_11_359c87ada34d682bae45.mp4 +2025-01-18 20:23:21,369 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:21,733 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:23:22,378 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:22,733 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:23,378 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:23,734 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:23,853 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:24,383 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:25,388 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:26,438 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:26,735 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:27,413 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:28,398 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:28,736 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:29,087 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:29,418 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:30,397 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:30,736 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:23:30,736 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:23:30,737 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:23:30,737 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:23:30,737 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:23:30,737 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:31,401 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:32,404 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:32,738 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:33,406 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:33,645 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:23:34,021 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:23:34,021 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:23:34,167 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:34,389 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:35,397 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:36,424 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:36,739 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:37,425 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:38,401 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:39,167 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:39,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:40,391 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:40,741 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:41,391 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:42,391 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:43,391 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:44,391 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:45,079 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:45,391 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:46,391 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:46,743 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:23:47,392 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:48,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:49,008 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:23:49,008 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:23:49,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:50,170 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:50,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:51,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:52,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:53,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:54,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:55,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:56,079 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:23:56,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:57,393 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:58,394 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:23:59,394 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:00,395 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:01,395 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:02,079 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:02,395 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:03,395 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:03,646 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:24:04,008 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:24:04,009 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:24:04,395 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:05,398 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:06,426 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:06,752 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:07,317 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:07,422 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:07,530 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:24:07,530 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_50069bff80de5f046230.mp4 with policy now +2025-01-18 20:24:07,539 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:24:07,539 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:24:07,539 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:24:07,539 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:24:07,752 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:24:07,752 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/12_eval_12_50069bff80de5f046230.mp4 +2025-01-18 20:24:07,752 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:24:07,787 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/xg2iymz1-media/videos/eval/12_eval_12_50069bff80de5f046230.mp4 +2025-01-18 20:24:08,044 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/7cxjiq6y-media/videos/eval/12_eval_12_50069bff80de5f046230.mp4 +2025-01-18 20:24:08,427 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:08,752 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:09,429 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:09,753 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:10,431 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:11,428 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:12,420 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:12,754 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:13,237 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:13,450 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:14,415 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:15,452 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:16,460 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:16,756 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:24:16,756 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:16,790 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:24:16,790 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:24:16,791 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:24:17,448 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:17,757 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:24:17,757 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:24:18,437 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:18,757 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:18,913 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:19,026 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:24:19,026 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:24:19,445 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:20,446 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:21,438 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:22,456 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:22,759 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:23,450 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:23,977 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:24,459 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:25,475 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:26,473 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:27,456 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:28,456 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:28,761 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:29,089 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:29,456 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:30,456 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:31,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:32,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:33,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:33,646 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:24:34,009 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:24:34,009 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:24:34,152 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:34,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:35,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:36,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:37,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:38,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:39,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:40,089 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:40,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:41,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:42,457 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:43,458 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:44,458 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:45,089 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:45,458 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:46,458 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:47,458 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:48,458 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:49,009 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:24:49,010 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:24:49,458 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:50,089 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:50,458 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:51,459 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:52,459 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:53,460 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:54,487 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:54,772 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:55,458 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:24:55,496 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:55,772 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/13_eval_13_114ad2ddaaeca4cacdd5.mp4 +2025-01-18 20:24:55,772 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:24:55,784 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:24:55,784 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_114ad2ddaaeca4cacdd5.mp4 with policy now +2025-01-18 20:24:55,799 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:24:55,799 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:24:55,799 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:24:55,799 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:24:56,052 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/zry9fzab-media/videos/eval/13_eval_13_114ad2ddaaeca4cacdd5.mp4 +2025-01-18 20:24:56,309 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/jsjkzokd-media/videos/eval/13_eval_13_114ad2ddaaeca4cacdd5.mp4 +2025-01-18 20:24:56,511 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:56,772 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:24:56,773 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:57,507 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:57,773 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:24:58,518 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:24:59,519 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:00,496 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:00,774 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:01,341 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:01,508 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:02,491 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:03,545 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:03,646 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:25:03,775 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:25:03,852 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:25:03,852 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:25:03,862 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:25:04,027 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/mdilzrm7-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:25:04,056 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:25:04,056 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:25:04,522 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:04,776 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:25:04,776 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:25:04,776 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:05,503 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:06,505 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:06,776 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:06,797 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:07,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:08,542 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:08,777 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:09,518 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:10,513 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:11,520 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:12,104 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:12,514 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:13,521 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:14,528 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:15,513 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:16,513 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:16,781 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:17,342 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:17,513 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:18,514 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:19,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:25:19,016 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:25:19,515 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:20,515 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:21,515 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:22,515 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:23,129 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:23,515 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:24,515 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:25,515 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:26,515 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:27,515 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:28,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:29,095 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:29,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:30,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:31,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:32,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:33,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:33,647 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:25:34,015 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:25:34,016 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:25:34,176 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:34,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:35,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:36,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:37,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:38,516 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:39,517 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:40,096 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:40,517 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:41,517 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:42,528 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:42,792 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:43,329 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:25:43,329 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_d43b1e73114608f44587.mp4 with policy now +2025-01-18 20:25:43,359 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:25:43,360 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:25:43,360 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:25:43,360 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:25:43,526 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:43,596 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/spkax966-media/videos/eval/14_eval_14_d43b1e73114608f44587.mp4 +2025-01-18 20:25:43,792 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:25:43,792 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/14_eval_14_d43b1e73114608f44587.mp4 +2025-01-18 20:25:43,792 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:25:44,065 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/ab6uze5e-media/videos/eval/14_eval_14_d43b1e73114608f44587.mp4 +2025-01-18 20:25:44,546 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:44,792 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:45,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:45,565 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:45,793 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:46,533 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:47,557 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:48,533 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:48,794 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:49,026 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:25:49,027 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:25:49,541 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:50,168 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:50,533 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:50,795 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:51,551 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:52,548 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:53,554 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:53,796 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:25:53,917 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:25:53,917 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:25:53,918 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:25:54,575 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:54,797 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:25:54,797 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:55,233 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:25:55,555 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:56,555 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:57,566 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:58,583 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:25:58,798 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:25:59,599 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:00,589 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:01,110 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:01,585 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:02,598 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:03,600 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:03,647 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:26:04,032 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:26:04,033 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:26:04,600 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:05,641 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:06,584 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:06,802 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:07,064 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:07,584 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:08,584 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:09,584 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:10,584 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:11,585 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:12,585 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:12,804 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:13,037 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:13,585 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:14,585 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:15,585 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:16,585 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:17,586 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:18,241 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:18,586 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:18,807 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:19,023 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:26:19,023 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:26:19,586 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:20,586 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:21,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:22,586 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:23,586 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:24,100 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:24,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:25,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:26,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:26,810 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:27,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:28,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:28,811 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:29,100 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:29,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:30,588 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:30,812 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:31,588 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:32,588 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:33,615 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:33,648 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:26:34,032 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:26:34,032 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:26:34,200 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:26:34,200 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:34,200 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_a134ab49305f64bfef91.mp4 with policy now +2025-01-18 20:26:34,202 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:26:34,203 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:26:34,203 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:26:34,203 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:26:34,470 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/lzxnvvl1-media/videos/eval/15_eval_15_a134ab49305f64bfef91.mp4 +2025-01-18 20:26:34,592 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:34,814 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:26:34,814 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:34,814 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/15_eval_15_a134ab49305f64bfef91.mp4 +2025-01-18 20:26:34,814 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:26:35,093 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/47xyupsk-media/videos/eval/15_eval_15_a134ab49305f64bfef91.mp4 +2025-01-18 20:26:35,621 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:35,814 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:36,624 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:37,650 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:38,618 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:38,815 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:39,636 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:40,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:40,626 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:41,637 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:42,624 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:43,646 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:43,818 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:44,670 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:45,125 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:45,638 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:46,665 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:47,675 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:48,663 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:49,060 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:26:49,060 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:26:49,673 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:49,820 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:26:49,986 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:26:49,986 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:26:49,986 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:26:50,666 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:50,821 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:26:50,821 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:26:50,922 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:51,671 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:51,821 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:52,692 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:53,686 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:53,822 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:54,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:55,694 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:55,823 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:26:56,131 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:26:56,692 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:57,695 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:58,726 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:26:59,715 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:00,745 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:01,142 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:01,722 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:02,720 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:03,648 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:27:03,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:03,826 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:04,025 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:27:04,025 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:27:04,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:05,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:05,827 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:06,187 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:06,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:07,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:08,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:09,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:10,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:11,698 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:12,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:12,699 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:13,699 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:14,699 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:15,699 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:16,699 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:17,699 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:18,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:18,699 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:19,025 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:27:19,025 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:27:19,699 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:20,700 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:21,700 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:22,700 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:23,163 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:23,700 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:24,700 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:25,700 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:25,836 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:26,701 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:27,701 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:28,701 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:29,119 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:29,701 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:30,725 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:30,974 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:27:30,974 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_fba70f3485b0b97a5e73.mp4 with policy now +2025-01-18 20:27:30,989 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:27:30,990 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:27:30,990 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:27:30,990 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:27:31,245 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/stvknmdz-media/videos/eval/16_eval_16_fba70f3485b0b97a5e73.mp4 +2025-01-18 20:27:31,743 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:31,838 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:27:31,838 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:31,839 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/16_eval_16_fba70f3485b0b97a5e73.mp4 +2025-01-18 20:27:31,839 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:27:32,130 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/quw091zt-media/videos/eval/16_eval_16_fba70f3485b0b97a5e73.mp4 +2025-01-18 20:27:32,742 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:33,649 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:27:33,731 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:33,839 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:34,039 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:27:34,040 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:27:34,180 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:34,734 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:35,760 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:36,749 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:37,733 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:37,841 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:38,770 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:39,762 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:39,842 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:40,051 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:40,778 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:40,842 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:27:41,050 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:27:41,050 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:27:41,050 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:27:41,102 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/52wmha3p-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:27:41,780 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:41,843 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:27:41,843 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:42,756 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:43,753 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:43,843 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:44,749 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:45,557 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:45,775 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:46,763 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:47,770 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:47,845 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:48,767 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:49,057 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:27:49,057 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:27:49,763 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:50,778 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:51,149 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:51,763 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:52,768 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:53,776 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:54,757 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:55,757 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:55,849 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:27:56,757 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:57,137 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:27:57,757 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:58,757 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:27:59,757 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:00,757 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:01,758 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:02,758 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:03,137 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:03,649 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:28:03,758 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:04,028 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:28:04,028 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:28:04,758 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:05,758 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:06,758 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:07,758 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:08,138 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:08,758 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:09,758 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:10,759 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:11,759 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:11,856 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:28:12,759 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:13,157 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:13,759 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:14,759 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:15,759 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:16,762 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:17,761 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:18,761 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:19,028 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:28:19,029 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:28:19,141 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:19,761 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:20,761 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:21,774 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:21,995 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:28:21,995 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_70585fb0ca24437a0f9f.mp4 with policy now +2025-01-18 20:28:22,026 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:28:22,026 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:28:22,026 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:28:22,027 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:28:22,292 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/89wxzpv4-media/videos/eval/17_eval_17_70585fb0ca24437a0f9f.mp4 +2025-01-18 20:28:22,786 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:22,861 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:28:22,861 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/17_eval_17_70585fb0ca24437a0f9f.mp4 +2025-01-18 20:28:22,861 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:28:23,409 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/h75uibxj-media/videos/eval/17_eval_17_70585fb0ca24437a0f9f.mp4 +2025-01-18 20:28:23,798 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:23,861 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:28:24,193 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:24,806 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:25,811 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:26,802 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:27,816 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:27,863 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:28:28,808 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:29,194 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:29,813 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:29,864 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:28:30,109 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:28:30,110 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:28:30,130 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:28:30,854 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:30,864 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:28:31,825 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:31,864 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:28:32,847 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:33,650 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:28:33,823 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:34,035 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:28:34,036 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:28:34,845 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:35,162 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:35,834 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:35,866 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:28:36,852 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:37,857 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:38,877 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:39,886 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:40,184 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:40,841 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:41,841 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:42,841 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:43,841 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:43,870 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:28:44,841 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:45,841 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:46,160 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:46,841 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:47,841 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:48,842 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:49,030 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:28:49,031 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:28:49,842 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:50,842 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:51,161 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:51,842 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:52,843 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:53,842 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:54,843 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:55,843 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:56,843 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:57,161 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:28:57,843 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:58,843 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:28:59,843 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:00,843 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:01,844 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:02,161 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:02,844 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:03,650 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:29:03,844 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:04,031 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:29:04,031 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:29:04,844 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:05,844 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:06,844 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:07,161 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:07,873 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:08,874 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:09,461 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:29:09,461 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_666566ff2e12fdecef30.mp4 with policy now +2025-01-18 20:29:09,510 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:29:09,511 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:29:09,511 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:29:09,512 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:29:09,723 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/5xldh2p6-media/videos/eval/18_eval_18_666566ff2e12fdecef30.mp4 +2025-01-18 20:29:09,857 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:09,881 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:29:09,881 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:29:09,881 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/18_eval_18_666566ff2e12fdecef30.mp4 +2025-01-18 20:29:09,881 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:29:10,160 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/xx9bayku-media/videos/eval/18_eval_18_666566ff2e12fdecef30.mp4 +2025-01-18 20:29:10,860 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:11,882 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:29:11,891 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:12,569 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:12,868 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:13,887 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:14,890 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:15,884 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:29:15,927 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:16,899 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:17,885 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:29:17,943 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:18,233 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:18,922 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:19,058 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:29:19,058 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:29:19,917 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:20,942 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:21,919 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:22,921 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:23,932 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:24,202 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:24,950 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:25,947 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:26,978 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:28,006 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:28,973 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:29,212 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:29,960 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:30,974 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:32,001 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:32,983 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:33,650 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:29:33,977 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:34,044 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:29:34,045 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:29:34,976 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:35,209 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:35,983 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:36,999 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:38,012 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:39,028 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:40,010 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:40,257 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:41,028 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:42,045 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:43,046 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:44,033 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:45,038 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:46,075 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:46,232 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:47,068 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:48,064 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:49,047 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:49,059 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:29:49,060 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:29:50,056 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:51,062 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:52,067 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:52,231 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:53,059 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:54,114 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:55,099 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:56,065 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:56,901 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:29:57,049 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:57,213 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:29:57,213 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:29:57,213 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:29:57,902 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:29:57,902 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:29:58,044 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:29:58,085 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:59,076 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:29:59,903 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:00,075 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:01,055 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:02,090 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:03,075 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:03,525 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:03,651 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:30:03,905 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:04,048 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:30:04,048 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:30:04,111 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:05,122 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:06,111 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:07,148 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:08,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:09,121 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:09,243 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:10,137 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:11,106 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:12,134 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:13,112 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:13,909 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:14,112 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:15,112 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:15,229 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:15,910 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:16,112 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:17,112 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:18,112 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:19,040 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:30:19,040 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:30:19,112 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:20,112 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:20,230 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:21,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:22,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:23,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:24,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:25,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:25,230 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:26,113 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:27,114 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:28,115 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:29,114 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:30,114 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:30,679 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:31,114 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:32,114 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:33,114 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:33,651 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:30:33,918 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:34,040 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:30:34,041 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:30:34,115 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:35,115 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:35,919 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:36,116 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:36,482 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:37,117 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:38,116 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:39,116 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:39,921 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:40,124 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:41,155 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:41,221 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:30:41,221 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:30:41,221 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_d8301c722455e230f538.mp4 with policy now +2025-01-18 20:30:41,222 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:30:41,222 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:41,223 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:41,504 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/eo65la2f-media/videos/eval/19_eval_19_d8301c722455e230f538.mp4 +2025-01-18 20:30:41,922 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:30:41,922 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:41,922 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/19_eval_19_d8301c722455e230f538.mp4 +2025-01-18 20:30:41,922 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval +2025-01-18 20:30:42,156 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:42,207 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/eglbeqc9-media/videos/eval/19_eval_19_d8301c722455e230f538.mp4 +2025-01-18 20:30:42,272 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:43,158 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:43,923 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:44,180 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:45,192 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:46,174 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:47,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:47,925 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:47,941 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:48,155 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:49,141 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:30:49,141 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:30:49,164 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:49,926 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:50,168 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:50,926 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:30:51,198 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:51,277 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:30:51,278 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:51,278 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:51,926 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:30:51,927 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:30:52,069 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 20:30:52,070 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 20:30:52,218 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:52,452 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:30:52,594 INFO wandb-upload_3:2569743 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpna8v3b00 +2025-01-18 20:30:52,853 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 20:30:52,863 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 20:30:52,927 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:30:52,927 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/table/eval/success_rate_ci_20_54e2754b6d7019322e44.table.json +2025-01-18 20:30:52,927 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/table/eval/total_success_rate_ci_21_84d6d84980b416ef739f.table.json +2025-01-18 20:30:52,928 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media +2025-01-18 20:30:52,928 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/table/eval +2025-01-18 20:30:52,928 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/table +2025-01-18 20:30:52,993 INFO SenderThread:2569743 [sender.py:send_request_log_artifact():1525] logged artifact run-h3eacbh9-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTk4MDY1MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzA3OTMxMQ==', 'latestArtifact': None}} +2025-01-18 20:30:52,993 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:30:52,993 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_54e2754b6d7019322e44.table.json with policy now +2025-01-18 20:30:52,993 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:30:52,993 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:52,994 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:52,994 DEBUG SenderThread:2569743 [sender.py:send():379] send: telemetry +2025-01-18 20:30:52,994 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:52,994 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:52,995 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:52,995 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:52,996 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:52,996 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:52,996 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:52,996 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:52,997 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:52,997 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:52,997 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:52,998 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:52,998 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:52,998 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:52,999 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:52,999 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:52,999 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:53,000 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:53,000 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:53,000 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:53,001 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:53,001 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:53,001 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:53,001 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 20:30:53,177 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:53,208 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/ey8we0uj-media/table/eval/total_success_rate_ci_21_84d6d84980b416ef739f.table.json +2025-01-18 20:30:53,235 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/lkkoy9fb-media/table/eval/success_rate_ci_20_54e2754b6d7019322e44.table.json +2025-01-18 20:30:53,261 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/v8391x8x-events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:30:53,493 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/g0d69rld-media/table/eval/success_rate_ci_20_54e2754b6d7019322e44.table.json +2025-01-18 20:30:53,571 INFO wandb-upload_0:2569743 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpx2fn7a47 +2025-01-18 20:30:53,928 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:30:53,928 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:54,066 INFO SenderThread:2569743 [sender.py:send_request_log_artifact():1525] logged artifact run-h3eacbh9-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTk4MDY2Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzA3OTMxMw==', 'latestArtifact': None}} +2025-01-18 20:30:54,066 DEBUG SenderThread:2569743 [sender.py:send():379] send: files +2025-01-18 20:30:54,066 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_84d6d84980b416ef739f.table.json with policy now +2025-01-18 20:30:54,066 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:30:54,067 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:30:54,084 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:30:54,178 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:54,315 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /tmp/tmptubput2iwandb/73vv4p4t-media/table/eval/total_success_rate_ci_21_84d6d84980b416ef739f.table.json +2025-01-18 20:30:54,928 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:30:55,178 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:55,929 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:30:56,178 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:57,178 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:58,178 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:30:58,245 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:30:59,178 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:00,178 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:01,178 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:02,179 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:03,179 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:03,245 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:31:03,652 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:31:04,050 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 20:31:04,050 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: stop_status +2025-01-18 20:31:04,179 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:05,179 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:06,179 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:07,179 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:08,179 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:08,245 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:31:09,179 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:10,180 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:11,180 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:12,180 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:13,180 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:14,180 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:14,245 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:31:15,093 DEBUG SenderThread:2569743 [sender.py:send():379] send: exit +2025-01-18 20:31:15,093 INFO SenderThread:2569743 [sender.py:send_exit():586] handling exit code: 0 +2025-01-18 20:31:15,094 INFO SenderThread:2569743 [sender.py:send_exit():588] handling runtime: 1031 +2025-01-18 20:31:15,094 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:31:15,094 INFO SenderThread:2569743 [sender.py:send_exit():594] send defer +2025-01-18 20:31:15,094 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:15,094 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-18 20:31:15,094 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:15,095 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-18 20:31:15,095 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 1 +2025-01-18 20:31:15,095 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:15,095 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-18 20:31:15,095 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:15,095 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-18 20:31:15,095 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 2 +2025-01-18 20:31:15,095 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:15,095 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-18 20:31:15,095 INFO HandlerThread:2569743 [system_monitor.py:finish():203] Stopping system monitor +2025-01-18 20:31:15,095 DEBUG SystemMonitor:2569743 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-18 20:31:15,095 DEBUG SystemMonitor:2569743 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-18 20:31:15,096 INFO HandlerThread:2569743 [interfaces.py:finish():200] Joined cpu monitor +2025-01-18 20:31:15,096 INFO HandlerThread:2569743 [interfaces.py:finish():200] Joined disk monitor +2025-01-18 20:31:15,164 INFO HandlerThread:2569743 [interfaces.py:finish():200] Joined gpu monitor +2025-01-18 20:31:15,165 INFO HandlerThread:2569743 [interfaces.py:finish():200] Joined memory monitor +2025-01-18 20:31:15,165 INFO HandlerThread:2569743 [interfaces.py:finish():200] Joined network monitor +2025-01-18 20:31:15,165 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:15,165 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-18 20:31:15,165 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 3 +2025-01-18 20:31:15,165 DEBUG SenderThread:2569743 [sender.py:send():379] send: stats +2025-01-18 20:31:15,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:15,166 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-18 20:31:15,166 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:15,166 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-18 20:31:15,166 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 4 +2025-01-18 20:31:15,166 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:15,166 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-18 20:31:15,939 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:31:17,940 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:31:21,287 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:31:21,287 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:31:21,287 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: keepalive +2025-01-18 20:31:21,289 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:21,289 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-18 20:31:21,289 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 5 +2025-01-18 20:31:21,289 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:31:21,289 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:21,289 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-18 20:31:21,291 DEBUG SenderThread:2569743 [sender.py:send():379] send: history +2025-01-18 20:31:21,292 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: summary_record +2025-01-18 20:31:21,292 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:31:21,292 DEBUG SenderThread:2569743 [sender.py:send():379] send: summary +2025-01-18 20:31:21,293 INFO SenderThread:2569743 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 20:31:21,293 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:21,293 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-18 20:31:21,293 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 6 +2025-01-18 20:31:21,293 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:21,293 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-18 20:31:21,293 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:21,293 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-18 20:31:21,297 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 20:31:21,426 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 7 +2025-01-18 20:31:21,426 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:21,426 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-18 20:31:21,426 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:21,426 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-18 20:31:21,942 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:31:21,942 INFO Thread-12 :2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/config.yaml +2025-01-18 20:31:22,095 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:31:23,397 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 8 +2025-01-18 20:31:23,397 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:31:23,398 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:23,398 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-18 20:31:23,398 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:23,398 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-18 20:31:23,398 INFO SenderThread:2569743 [job_builder.py:build():440] Attempting to build job artifact +2025-01-18 20:31:23,400 INFO SenderThread:2569743 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-18 20:31:23,402 INFO SenderThread:2569743 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-18 20:31:23,416 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 9 +2025-01-18 20:31:23,416 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:23,416 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-18 20:31:23,416 DEBUG SenderThread:2569743 [sender.py:send():379] send: artifact +2025-01-18 20:31:23,631 INFO SenderThread:2569743 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1MTk1NDMyOQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1MTk1NDMyOQ==', 'versionIndex': 449}}} +2025-01-18 20:31:23,631 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:23,631 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-18 20:31:23,631 INFO SenderThread:2569743 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-18 20:31:23,944 INFO SenderThread:2569743 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:31:23,944 INFO SenderThread:2569743 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files +2025-01-18 20:31:23,944 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/config.yaml config.yaml +2025-01-18 20:31:23,944 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 events.out.tfevents.1737249243.leonmkim-ROG-Strix-G15CS-G15CS.2569579.0 +2025-01-18 20:31:23,944 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json wandb-summary.json +2025-01-18 20:31:23,944 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/requirements.txt requirements.txt +2025-01-18 20:31:23,944 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-metadata.json wandb-metadata.json +2025-01-18 20:31:23,944 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 20:31:23,945 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log output.log +2025-01-18 20:31:23,946 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/diff.patch diff.patch +2025-01-18 20:31:23,950 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-18 20:31:23,950 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/19_eval_19_d8301c722455e230f538.mp4 media/videos/eval/19_eval_19_d8301c722455e230f538.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/16_eval_16_fba70f3485b0b97a5e73.mp4 media/videos/eval/16_eval_16_fba70f3485b0b97a5e73.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/12_eval_12_50069bff80de5f046230.mp4 media/videos/eval/12_eval_12_50069bff80de5f046230.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/9_eval_9_45f46258144727274eca.mp4 media/videos/eval/9_eval_9_45f46258144727274eca.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/8_eval_8_c1b631f94f507d0b8029.mp4 media/videos/eval/8_eval_8_c1b631f94f507d0b8029.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/4_eval_4_73cfbcf4697dd02a1704.mp4 media/videos/eval/4_eval_4_73cfbcf4697dd02a1704.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/18_eval_18_666566ff2e12fdecef30.mp4 media/videos/eval/18_eval_18_666566ff2e12fdecef30.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/6_eval_6_6f92e10ddc5700f95864.mp4 media/videos/eval/6_eval_6_6f92e10ddc5700f95864.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/5_eval_5_8eee7b4209c8cc001b29.mp4 media/videos/eval/5_eval_5_8eee7b4209c8cc001b29.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/17_eval_17_70585fb0ca24437a0f9f.mp4 media/videos/eval/17_eval_17_70585fb0ca24437a0f9f.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/11_eval_11_359c87ada34d682bae45.mp4 media/videos/eval/11_eval_11_359c87ada34d682bae45.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/13_eval_13_114ad2ddaaeca4cacdd5.mp4 media/videos/eval/13_eval_13_114ad2ddaaeca4cacdd5.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/7_eval_7_ec916f9f1cb86bb268d4.mp4 media/videos/eval/7_eval_7_ec916f9f1cb86bb268d4.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/2_eval_2_35d91a735aea17ce0770.mp4 media/videos/eval/2_eval_2_35d91a735aea17ce0770.mp4 +2025-01-18 20:31:23,951 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/15_eval_15_a134ab49305f64bfef91.mp4 media/videos/eval/15_eval_15_a134ab49305f64bfef91.mp4 +2025-01-18 20:31:23,952 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/14_eval_14_d43b1e73114608f44587.mp4 media/videos/eval/14_eval_14_d43b1e73114608f44587.mp4 +2025-01-18 20:31:23,952 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/10_eval_10_dbfaa8aaf0c838051064.mp4 media/videos/eval/10_eval_10_dbfaa8aaf0c838051064.mp4 +2025-01-18 20:31:23,952 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/1_eval_1_c93c245f94bccf988bf8.mp4 media/videos/eval/1_eval_1_c93c245f94bccf988bf8.mp4 +2025-01-18 20:31:23,952 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/3_eval_3_c4ac9168429d53521dfd.mp4 media/videos/eval/3_eval_3_c4ac9168429d53521dfd.mp4 +2025-01-18 20:31:23,952 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/videos/eval/0_eval_0_958873708057cb5c96d0.mp4 media/videos/eval/0_eval_0_958873708057cb5c96d0.mp4 +2025-01-18 20:31:23,952 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/table/eval/total_success_rate_ci_21_84d6d84980b416ef739f.table.json media/table/eval/total_success_rate_ci_21_84d6d84980b416ef739f.table.json +2025-01-18 20:31:23,952 INFO SenderThread:2569743 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/media/table/eval/success_rate_ci_20_54e2754b6d7019322e44.table.json media/table/eval/success_rate_ci_20_54e2754b6d7019322e44.table.json +2025-01-18 20:31:23,952 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 10 +2025-01-18 20:31:23,952 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:23,952 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-18 20:31:23,952 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:23,953 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-18 20:31:23,953 INFO SenderThread:2569743 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 20:31:24,095 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:31:24,096 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:31:24,116 INFO wandb-upload_0:2569743 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/wandb-summary.json +2025-01-18 20:31:24,128 INFO wandb-upload_3:2569743 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/config.yaml +2025-01-18 20:31:24,136 INFO wandb-upload_1:2569743 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/requirements.txt +2025-01-18 20:31:24,186 INFO wandb-upload_2:2569743 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/files/output.log +2025-01-18 20:31:24,387 INFO Thread-11 (_thread_body):2569743 [sender.py:transition_state():614] send defer: 11 +2025-01-18 20:31:24,387 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:24,387 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-18 20:31:24,387 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:24,387 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-18 20:31:24,387 INFO SenderThread:2569743 [file_pusher.py:join():175] waiting for file pusher +2025-01-18 20:31:24,390 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 12 +2025-01-18 20:31:24,391 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:24,391 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-18 20:31:24,391 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:24,391 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-18 20:31:24,391 INFO SenderThread:2569743 [file_stream.py:finish():601] file stream finish called +2025-01-18 20:31:24,585 INFO SenderThread:2569743 [file_stream.py:finish():605] file stream finish is done +2025-01-18 20:31:24,585 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 13 +2025-01-18 20:31:24,585 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:24,585 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-18 20:31:24,586 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:24,586 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-18 20:31:24,586 INFO SenderThread:2569743 [sender.py:transition_state():614] send defer: 14 +2025-01-18 20:31:24,586 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: defer +2025-01-18 20:31:24,586 INFO HandlerThread:2569743 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-18 20:31:24,586 DEBUG SenderThread:2569743 [sender.py:send():379] send: final +2025-01-18 20:31:24,586 DEBUG SenderThread:2569743 [sender.py:send():379] send: footer +2025-01-18 20:31:24,586 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: defer +2025-01-18 20:31:24,586 INFO SenderThread:2569743 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-18 20:31:24,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:31:24,587 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:31:24,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: server_info +2025-01-18 20:31:24,587 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: server_info +2025-01-18 20:31:24,587 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 20:31:24,589 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: get_summary +2025-01-18 20:31:24,589 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-18 20:31:24,590 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 20:31:24,645 DEBUG SenderThread:2569743 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 20:31:24,645 INFO MainThread:2569743 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-18 20:31:24,646 INFO MainThread:2569743 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-18 20:31:24,646 INFO MainThread:2569743 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-18 20:31:24,646 DEBUG HandlerThread:2569743 [handler.py:handle_request():158] handle_request: shutdown +2025-01-18 20:31:24,646 INFO HandlerThread:2569743 [handler.py:finish():882] shutting down handler +2025-01-18 20:31:25,589 INFO WriterThread:2569743 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/run-h3eacbh9.wandb +2025-01-18 20:31:25,645 INFO SenderThread:2569743 [sender.py:finish():1615] shutting down sender +2025-01-18 20:31:25,645 INFO SenderThread:2569743 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 20:31:25,645 INFO SenderThread:2569743 [file_pusher.py:join():175] waiting for file pusher diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/logs/debug.log b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e02dc0e2bc4f8b61dfcce249e1d4177d4286a569 --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-18 20:14:03,259 INFO MainThread:2569579 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-18 20:14:03,259 INFO MainThread:2569579 [wandb_setup.py:_flush():76] Configure stats pid to 2569579 +2025-01-18 20:14:03,259 INFO MainThread:2569579 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-18 20:14:03,259 INFO MainThread:2569579 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/settings +2025-01-18 20:14:03,260 INFO MainThread:2569579 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-18 20:14:03,260 INFO MainThread:2569579 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-18 20:14:03,260 INFO MainThread:2569579 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-18 20:14:03,260 INFO MainThread:2569579 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-18 20:14:03,260 INFO MainThread:2569579 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/logs/debug.log +2025-01-18 20:14:03,260 INFO MainThread:2569579 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/logs/debug-internal.log +2025-01-18 20:14:03,260 INFO MainThread:2569579 [wandb_init.py:init():569] calling init triggers +2025-01-18 20:14:03,261 INFO MainThread:2569579 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_0', 'wandb_notes': '3465_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9'} +2025-01-18 20:14:03,261 INFO MainThread:2569579 [wandb_init.py:init():619] starting backend +2025-01-18 20:14:03,261 INFO MainThread:2569579 [wandb_init.py:init():623] setting up manager +2025-01-18 20:14:03,265 INFO MainThread:2569579 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-18 20:14:03,266 INFO MainThread:2569579 [wandb_init.py:init():631] backend started and connected +2025-01-18 20:14:03,278 INFO MainThread:2569579 [wandb_init.py:init():720] updated telemetry +2025-01-18 20:14:03,284 INFO MainThread:2569579 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-18 20:14:03,542 INFO MainThread:2569579 [wandb_run.py:_on_init():2435] communicating current version +2025-01-18 20:14:03,616 INFO MainThread:2569579 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-18 20:14:03,617 INFO MainThread:2569579 [wandb_init.py:init():804] starting run threads in backend +2025-01-18 20:14:03,985 INFO MainThread:2569579 [wandb_run.py:_console_start():2413] atexit reg +2025-01-18 20:14:03,985 INFO MainThread:2569579 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-18 20:14:03,985 INFO MainThread:2569579 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-18 20:14:03,985 INFO MainThread:2569579 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-18 20:14:03,987 INFO MainThread:2569579 [wandb_init.py:init():847] run started, returning control to user process +2025-01-18 20:14:03,987 INFO MainThread:2569579 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/h3eacbh9/tb, True +2025-01-18 20:14:09,077 INFO MainThread:2569579 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-18 20:31:25,647 WARNING MsgRouterThr:2569579 [router.py:message_loop():77] message_loop has been closed diff --git a/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/run-h3eacbh9.wandb b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/run-h3eacbh9.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5932bfef2221800aa91fd803287222a2ac9b3aee --- /dev/null +++ b/h3eacbh9/wandb/run-20250118_201403-h3eacbh9/run-h3eacbh9.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb862ec463b2a5f17218cc40b0b00813fc38510dffc89e622e9d3a34cd518725 +size 237711 diff --git a/q1ti958r/episode_rosbags/episode_17_2025-01-18-19-04-12.bag b/q1ti958r/episode_rosbags/episode_17_2025-01-18-19-04-12.bag new file mode 100644 index 0000000000000000000000000000000000000000..5addce639cea19934ebc5a726123dab6726c16af --- /dev/null +++ b/q1ti958r/episode_rosbags/episode_17_2025-01-18-19-04-12.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:270921f27324b36cec1928f01887895ff626f699e9116bea588dcc1ee25ce9e6 +size 1411371657 diff --git a/q1ti958r/episode_rosbags/episode_1_2025-01-18-18-51-19.bag b/q1ti958r/episode_rosbags/episode_1_2025-01-18-18-51-19.bag new file mode 100644 index 0000000000000000000000000000000000000000..2ee9b52d74a07c1e32042116ca5b139895c32d8b --- /dev/null +++ b/q1ti958r/episode_rosbags/episode_1_2025-01-18-18-51-19.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3256665a5df9af866f0901d279fa22b1f28b42f256bf28d2fa1c0786c478327 +size 1411754884