diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..2a99eecca72910700a294fda2b24774f0c718fa4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,46 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/0_eval_0_294304c5db5cd37da633.mp4 filter=lfs diff=lfs merge=lfs -text +niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/3_eval_3_48a28150d9236694be84.mp4 filter=lfs diff=lfs merge=lfs -text +niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/4_eval_4_ae99ac981fb1172cf28a.mp4 filter=lfs diff=lfs merge=lfs -text +niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/1_eval_1_9e0e7530b69fb1fd581f.mp4 filter=lfs diff=lfs merge=lfs -text +niwmozqn/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/2_eval_2_a1afdd9f2c1c0e92713f.mp4 filter=lfs diff=lfs merge=lfs -text +niwmozqn/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +niwmozqn/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +niwmozqn/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +niwmozqn/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/4_eval_4_2da638a425088a67de89.mp4 filter=lfs diff=lfs merge=lfs -text +bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/0_eval_0_4df030dc74284b36be4e.mp4 filter=lfs diff=lfs merge=lfs -text +bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/3_eval_3_db8eec7ff63d941b6e62.mp4 filter=lfs diff=lfs merge=lfs -text +bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/2_eval_2_c9be307d2b8f293d1a04.mp4 filter=lfs diff=lfs merge=lfs -text +bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/1_eval_1_1d43c0fcec8d2ec15ec2.mp4 filter=lfs diff=lfs merge=lfs -text +bufkm1dk/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bufkm1dk/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bufkm1dk/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bufkm1dk/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bufkm1dk/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bufkm1dk/episode_rosbags/episode_3_2024-12-31-21-57-18.bag filter=lfs diff=lfs merge=lfs -text +bufkm1dk/episode_rosbags/episode_1_2024-12-31-21-56-11.bag filter=lfs diff=lfs merge=lfs -text +71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 filter=lfs diff=lfs merge=lfs -text +71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 filter=lfs diff=lfs merge=lfs -text +71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 filter=lfs diff=lfs merge=lfs -text +71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 filter=lfs diff=lfs merge=lfs -text +71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 filter=lfs diff=lfs merge=lfs -text +71icjjya/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +71icjjya/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +71icjjya/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +71icjjya/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +71icjjya/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bufkm1dk/episode_rosbags/episode_2_2024-12-31-21-56-46.bag filter=lfs diff=lfs merge=lfs -text +bufkm1dk/episode_rosbags/episode_4_2024-12-31-21-57-51.bag filter=lfs diff=lfs merge=lfs -text +bufkm1dk/episode_rosbags/episode_0_2024-12-31-21-55-35.bag filter=lfs diff=lfs merge=lfs -text +3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/2_eval_2_69a94bb80ce32994374e.mp4 filter=lfs diff=lfs merge=lfs -text +3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/4_eval_4_72049e080103157b9708.mp4 filter=lfs diff=lfs merge=lfs -text +3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/3_eval_3_e19838f640f8887956ae.mp4 filter=lfs diff=lfs merge=lfs -text +3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/0_eval_0_c2ff683945f18b252db9.mp4 filter=lfs diff=lfs merge=lfs -text +3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/1_eval_1_b0a9c00b6ce785ea7915.mp4 filter=lfs diff=lfs merge=lfs -text +3bbvaqak/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3bbvaqak/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3bbvaqak/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/3bbvaqak/episode_rosbags/aligned_depth_to_color_K.npy b/3bbvaqak/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/3bbvaqak/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/3bbvaqak/episode_rosbags/cam_tf_world.npy b/3bbvaqak/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/3bbvaqak/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/3bbvaqak/episode_rosbags/color_K.npy b/3bbvaqak/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/3bbvaqak/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/3bbvaqak/episode_rosbags/depth_K.npy b/3bbvaqak/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/3bbvaqak/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/3bbvaqak/eval_robot.log b/3bbvaqak/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..51355e6ee6c0a43d6745ad467f3ae142c8436faa --- /dev/null +++ b/3bbvaqak/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-31 22:33:46,371][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-31 22:33:46,373][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/3bbvaqak/eval_video/1_eval.mp4 b/3bbvaqak/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..87e3871cd3f9ec3db4870c8f0250e762a550a85a --- /dev/null +++ b/3bbvaqak/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0a9c00b6ce785ea7915153a57e3c3a18351caca9074eb5410bdfcd1d23fb9ce +size 1498557 diff --git a/3bbvaqak/eval_video/3_eval.mp4 b/3bbvaqak/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..85494cc50f8ef2461da6fffb832df46732001df5 --- /dev/null +++ b/3bbvaqak/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e19838f640f8887956aea499b2d301f845eb33dc9585229779be2198bf89afa6 +size 1541308 diff --git a/3bbvaqak/eval_video/4_eval.mp4 b/3bbvaqak/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..45f36f81c9fe6ec5abb73c2e21acc98efd026a19 --- /dev/null +++ b/3bbvaqak/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72049e080103157b9708135d69265581d5d9a1c8d4ca1e0db76e8dbac9047f0a +size 1558272 diff --git a/3bbvaqak/tb/events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 b/3bbvaqak/tb/events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 new file mode 100644 index 0000000000000000000000000000000000000000..e28249d9b73c958a46688b29068b304acc061a32 --- /dev/null +++ b/3bbvaqak/tb/events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1297a924d3edbb2da8bcd842d32970e946757b4e952bbc9ab7c18f0ba7682a85 +size 6539 diff --git a/3bbvaqak/wandb/debug-internal.log b/3bbvaqak/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..17cabfc6d32757766e7630f77278420ac474b6ac --- /dev/null +++ b/3bbvaqak/wandb/debug-internal.log @@ -0,0 +1,801 @@ +2024-12-31 22:33:52,169 INFO StreamThr :1759480 [internal.py:wandb_internal():85] W&B internal server running at pid: 1759480, started at: 2024-12-31 22:33:52.168929 +2024-12-31 22:33:52,170 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status +2024-12-31 22:33:52,183 INFO WriterThread:1759480 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/run-3bbvaqak.wandb +2024-12-31 22:33:52,183 DEBUG SenderThread:1759480 [sender.py:send():379] send: header +2024-12-31 22:33:52,190 DEBUG SenderThread:1759480 [sender.py:send():379] send: run +2024-12-31 22:33:52,386 INFO SenderThread:1759480 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files +2024-12-31 22:33:52,386 INFO SenderThread:1759480 [sender.py:_start_run_threads():1188] run started: 3bbvaqak with start time 1735702432.170474 +2024-12-31 22:33:52,402 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 22:33:52,402 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: check_version +2024-12-31 22:33:52,479 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 22:33:52,484 DEBUG HandlerThread:1759480 [system_info.py:__init__():26] System info init +2024-12-31 22:33:52,484 DEBUG HandlerThread:1759480 [system_info.py:__init__():41] System info init done +2024-12-31 22:33:52,484 INFO HandlerThread:1759480 [system_monitor.py:start():194] Starting system monitor +2024-12-31 22:33:52,484 INFO SystemMonitor:1759480 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 22:33:52,484 INFO HandlerThread:1759480 [system_monitor.py:probe():214] Collecting system info +2024-12-31 22:33:52,485 INFO SystemMonitor:1759480 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 22:33:52,485 INFO SystemMonitor:1759480 [interfaces.py:start():188] Started disk monitoring +2024-12-31 22:33:52,486 INFO SystemMonitor:1759480 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 22:33:52,487 INFO SystemMonitor:1759480 [interfaces.py:start():188] Started memory monitoring +2024-12-31 22:33:52,487 INFO SystemMonitor:1759480 [interfaces.py:start():188] Started network monitoring +2024-12-31 22:33:52,497 DEBUG HandlerThread:1759480 [system_info.py:probe():152] Probing system +2024-12-31 22:33:52,498 DEBUG HandlerThread:1759480 [system_info.py:_probe_git():137] Probing git +2024-12-31 22:33:52,503 DEBUG HandlerThread:1759480 [system_info.py:_probe_git():145] Probing git done +2024-12-31 22:33:52,503 DEBUG HandlerThread:1759480 [system_info.py:probe():200] Probing system done +2024-12-31 22:33:52,503 DEBUG HandlerThread:1759480 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T03:33:52.497155', 'startedAt': '2025-01-01T03:33:52.161539', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3560.59375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.004, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.64, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 560.3900833129883}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 22:33:52,503 INFO HandlerThread:1759480 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 22:33:52,503 INFO HandlerThread:1759480 [system_monitor.py:probe():227] Publishing system info +2024-12-31 22:33:52,503 DEBUG HandlerThread:1759480 [system_info.py:_save_code():44] Saving code +2024-12-31 22:33:52,505 DEBUG HandlerThread:1759480 [system_info.py:_save_code():67] Saving code done +2024-12-31 22:33:52,505 DEBUG HandlerThread:1759480 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 22:33:52,567 DEBUG HandlerThread:1759480 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 22:33:52,567 INFO HandlerThread:1759480 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 22:33:52,570 DEBUG SenderThread:1759480 [sender.py:send():379] send: files +2024-12-31 22:33:52,570 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 22:33:52,570 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 22:33:52,570 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 22:33:52,800 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 22:33:52,801 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: python_packages +2024-12-31 22:33:52,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:33:52,802 DEBUG SenderThread:1759480 [sender.py:send():379] send: telemetry +2024-12-31 22:33:52,820 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/g166iolm-wandb-metadata.json +2024-12-31 22:33:52,831 INFO wandb-upload_2:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/iz96odku-diff.patch +2024-12-31 22:33:52,849 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:33:52,849 INFO HandlerThread:1759480 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/tb" + save: true +} +control { + relay_id: "bf3d693ffd2549f08dea2c0f264a209a" +} +_info { + stream_id: "3bbvaqak" +} + +2024-12-31 22:33:52,849 DEBUG HandlerThread:1759480 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 22:33:52,850 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:33:52,854 INFO wandb-upload_1:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/z57m9fzd-code/FISH/eval_robot.py +2024-12-31 22:33:53,004 DEBUG SenderThread:1759480 [sender.py:send():379] send: tbrecord +2024-12-31 22:33:53,004 DEBUG SenderThread:1759480 [sender.py:send():379] send: files +2024-12-31 22:33:53,004 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 with policy live +2024-12-31 22:33:53,188 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/dm0fawzm-events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:33:53,388 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/diff.patch +2024-12-31 22:33:53,388 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/code/FISH/eval_robot.py +2024-12-31 22:33:53,388 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:33:53,388 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:33:53,388 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-metadata.json +2024-12-31 22:33:53,388 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/requirements.txt +2024-12-31 22:33:53,389 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/code/FISH +2024-12-31 22:33:53,389 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/code +2024-12-31 22:33:53,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:33:54,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:33:55,387 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:33:55,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:33:56,746 DEBUG SenderThread:1759480 [sender.py:send():379] send: config +2024-12-31 22:33:56,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:33:57,757 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:33:57,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:33:58,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:33:59,388 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:33:59,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:00,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:01,389 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:01,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:02,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:02,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:03,389 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:03,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:04,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:05,390 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:05,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:06,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:07,391 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:07,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:34:07,801 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:34:07,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:07,910 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:08,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:09,391 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:09,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:10,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:11,392 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:11,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:12,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:13,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:13,804 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:14,804 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:15,394 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:15,804 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:16,804 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:17,804 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:18,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:18,804 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:19,804 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:20,805 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:21,805 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:22,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:34:22,801 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:34:22,845 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:23,805 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:23,922 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:24,396 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/config.yaml +2024-12-31 22:34:24,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:25,397 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:25,805 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:26,805 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:27,805 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:28,805 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:29,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:29,805 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:30,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:31,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:32,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:33,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:34,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:34,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:35,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:36,807 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:37,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:34:37,801 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:34:37,845 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:38,808 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:39,808 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:39,872 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:40,808 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:41,808 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:42,808 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:43,401 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:43,808 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:44,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:45,541 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:45,769 DEBUG SenderThread:1759480 [sender.py:send():379] send: files +2024-12-31 22:34:45,769 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c2ff683945f18b252db9.mp4 with policy now +2024-12-31 22:34:45,769 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:34:45,771 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:34:45,771 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:34:45,771 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:34:45,808 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:46,013 INFO wandb-upload_1:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/holiaaxl-media/videos/eval/0_eval_0_c2ff683945f18b252db9.mp4 +2024-12-31 22:34:46,402 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:34:46,403 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/0_eval_0_c2ff683945f18b252db9.mp4 +2024-12-31 22:34:46,403 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media +2024-12-31 22:34:46,403 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos +2024-12-31 22:34:46,403 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval +2024-12-31 22:34:46,683 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/9qjwdrvb-media/videos/eval/0_eval_0_c2ff683945f18b252db9.mp4 +2024-12-31 22:34:46,808 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:47,403 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:47,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:48,809 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:49,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:50,811 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:51,365 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:51,404 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:51,811 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:52,487 DEBUG SystemMonitor:1759480 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 22:34:52,488 DEBUG SenderThread:1759480 [sender.py:send():379] send: stats +2024-12-31 22:34:52,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:34:52,801 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:34:52,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:53,405 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:53,811 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:54,811 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:55,405 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:34:55,699 INFO wandb-upload_1:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/d371h7p7-events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:34:55,811 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:56,653 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:56,811 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:57,406 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:57,811 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:58,811 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:59,407 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:59,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:00,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:01,408 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:35:01,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:02,262 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:02,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:03,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:04,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:05,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:06,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:07,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:35:07,801 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:35:07,845 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:07,862 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:08,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:09,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:10,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:11,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:12,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:13,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:13,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:14,814 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:15,814 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:16,814 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:17,814 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:18,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:18,814 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:19,814 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:20,814 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:21,814 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:22,488 DEBUG SenderThread:1759480 [sender.py:send():379] send: stats +2024-12-31 22:35:22,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:35:22,801 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:35:22,849 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:23,815 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:23,880 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:24,815 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:25,815 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:26,816 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:27,816 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:28,816 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:29,363 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:29,816 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:30,816 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:31,417 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:35:31,816 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:32,817 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:33,817 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:34,572 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:34,820 DEBUG SenderThread:1759480 [sender.py:send():379] send: files +2024-12-31 22:35:34,820 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b0a9c00b6ce785ea7915.mp4 with policy now +2024-12-31 22:35:34,823 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:35:34,824 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:35:34,824 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:35:34,827 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:35:34,828 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:35,064 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/kofetkye-media/videos/eval/1_eval_1_b0a9c00b6ce785ea7915.mp4 +2024-12-31 22:35:35,418 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:35:35,418 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:35:35,418 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/1_eval_1_b0a9c00b6ce785ea7915.mp4 +2024-12-31 22:35:35,419 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval +2024-12-31 22:35:35,709 INFO wandb-upload_1:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/j629ah60-media/videos/eval/1_eval_1_b0a9c00b6ce785ea7915.mp4 +2024-12-31 22:35:35,829 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:36,830 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:37,419 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:35:37,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:35:37,812 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:35:37,853 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:38,832 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:39,420 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:35:39,827 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:40,413 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:40,824 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:41,832 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:42,829 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:43,421 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:35:43,828 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:44,828 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:45,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:45,828 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:46,831 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:47,839 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:48,839 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:49,834 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:50,815 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:50,830 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:51,834 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:52,489 DEBUG SenderThread:1759480 [sender.py:send():379] send: stats +2024-12-31 22:35:52,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:35:52,804 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:35:52,849 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:53,833 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:54,837 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:55,834 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:55,897 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:56,836 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:57,833 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:58,265 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:35:58,265 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:35:58,265 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:35:58,426 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:35:58,426 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:35:58,679 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/h3wua1zg-events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:35:58,838 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:59,426 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:35:59,842 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:00,842 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:01,261 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:01,841 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:02,840 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:03,428 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:03,841 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:05,035 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:05,428 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:06,027 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:06,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:07,027 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:07,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:36:07,802 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:36:08,027 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:09,027 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:10,027 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:11,027 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:11,890 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:12,027 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:13,028 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:13,432 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:14,028 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:15,028 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:16,028 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:17,028 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:17,805 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:18,028 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:19,028 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:19,434 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:20,029 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:21,029 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:21,434 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:22,030 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:22,489 DEBUG SenderThread:1759480 [sender.py:send():379] send: stats +2024-12-31 22:36:22,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:36:22,802 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:36:22,888 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:23,030 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:24,030 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:25,030 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:26,031 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:27,030 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:28,030 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:28,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:29,031 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:30,031 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:31,031 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:32,031 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:33,031 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:33,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:34,032 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:35,031 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:36,032 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:37,032 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:37,817 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:36:37,817 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:36:38,035 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:38,308 DEBUG SenderThread:1759480 [sender.py:send():379] send: files +2024-12-31 22:36:38,309 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_69a94bb80ce32994374e.mp4 with policy now +2024-12-31 22:36:38,313 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:36:38,314 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:36:38,314 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:36:38,314 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:36:38,440 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:36:38,440 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/2_eval_2_69a94bb80ce32994374e.mp4 +2024-12-31 22:36:38,440 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval +2024-12-31 22:36:38,547 INFO wandb-upload_1:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/d9zzwm60-media/videos/eval/2_eval_2_69a94bb80ce32994374e.mp4 +2024-12-31 22:36:38,827 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/s3pgh7ip-media/videos/eval/2_eval_2_69a94bb80ce32994374e.mp4 +2024-12-31 22:36:39,045 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:39,343 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:39,440 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:40,047 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:41,045 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:42,046 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:43,046 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:43,441 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:44,047 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:44,737 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:45,046 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:46,048 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:47,056 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:47,443 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:48,057 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:49,050 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:49,111 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:36:49,111 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:36:49,111 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:36:49,444 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:36:49,444 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:36:49,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:50,058 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:51,057 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:51,444 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:52,070 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:52,490 DEBUG SenderThread:1759480 [sender.py:send():379] send: stats +2024-12-31 22:36:52,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:36:52,807 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:36:53,063 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:53,445 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:54,067 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:55,059 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:55,321 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:55,446 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:56,062 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:57,060 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:57,446 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:58,060 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:59,061 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:00,061 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:00,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:01,061 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:02,061 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:03,062 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:04,062 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:05,062 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:05,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:06,063 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:07,063 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:07,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:37:07,803 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:37:08,063 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:09,063 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:10,063 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:10,920 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:11,063 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:12,063 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:13,064 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:14,064 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:15,064 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:15,453 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:37:16,064 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:16,494 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:17,064 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:18,066 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:19,064 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:20,065 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:21,065 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:21,815 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:22,065 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:22,490 DEBUG SenderThread:1759480 [sender.py:send():379] send: stats +2024-12-31 22:37:22,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:37:22,803 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:37:23,065 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:24,065 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:25,065 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:26,065 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:26,914 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:27,065 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:28,065 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:29,066 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:30,081 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:31,076 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:31,148 DEBUG SenderThread:1759480 [sender.py:send():379] send: files +2024-12-31 22:37:31,149 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_e19838f640f8887956ae.mp4 with policy now +2024-12-31 22:37:31,149 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:37:31,150 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:37:31,150 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:37:31,158 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:37:31,389 INFO wandb-upload_1:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/ns4oe1jc-media/videos/eval/3_eval_3_e19838f640f8887956ae.mp4 +2024-12-31 22:37:31,458 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:37:31,458 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:37:31,459 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/3_eval_3_e19838f640f8887956ae.mp4 +2024-12-31 22:37:31,459 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval +2024-12-31 22:37:31,745 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/mn74s48n-media/videos/eval/3_eval_3_e19838f640f8887956ae.mp4 +2024-12-31 22:37:32,071 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:32,185 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:33,076 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:33,459 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:37:34,080 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:35,078 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:35,460 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:37:36,075 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:37,076 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:37,497 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:37,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:37:37,803 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:37:38,077 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:39,078 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:39,461 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:37:40,087 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:41,094 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:42,087 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:42,617 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:43,084 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:43,174 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:37:43,175 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:37:43,175 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:37:43,462 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:37:43,462 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:37:43,463 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:37:43,705 INFO wandb-upload_1:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/mdfu3xib-events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:37:44,086 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:45,087 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:45,463 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:37:46,089 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:47,090 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:47,464 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:37:47,816 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:48,087 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:49,085 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:50,086 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:51,085 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:51,465 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:37:52,085 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:52,491 DEBUG SenderThread:1759480 [sender.py:send():379] send: stats +2024-12-31 22:37:52,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:37:52,804 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:37:52,934 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:53,087 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:54,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:55,087 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:56,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:57,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:58,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:58,815 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:59,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:00,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:01,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:01,468 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:02,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:03,089 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:03,815 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:38:04,089 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:05,089 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:05,470 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:06,089 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:07,089 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:07,804 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:38:07,804 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:38:08,090 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:08,907 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:38:09,090 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:10,090 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:11,090 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:12,090 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:13,090 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:14,091 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:14,815 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:38:15,091 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:16,091 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:17,091 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:18,091 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:19,091 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:19,815 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:38:20,092 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:21,091 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:22,094 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:22,491 DEBUG SenderThread:1759480 [sender.py:send():379] send: stats +2024-12-31 22:38:22,804 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:38:22,804 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:38:23,092 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:24,092 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:25,095 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:25,453 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:38:25,477 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:25,734 DEBUG SenderThread:1759480 [sender.py:send():379] send: files +2024-12-31 22:38:25,734 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_72049e080103157b9708.mp4 with policy now +2024-12-31 22:38:25,735 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:38:25,735 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:38:25,735 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:25,737 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:25,978 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/neu09czi-media/videos/eval/4_eval_4_72049e080103157b9708.mp4 +2024-12-31 22:38:26,095 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:26,477 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:38:26,477 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/4_eval_4_72049e080103157b9708.mp4 +2024-12-31 22:38:26,477 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval +2024-12-31 22:38:26,799 INFO wandb-upload_1:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/5yizbm4v-media/videos/eval/4_eval_4_72049e080103157b9708.mp4 +2024-12-31 22:38:27,123 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:27,477 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:28,102 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:29,116 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:30,106 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:31,118 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:31,445 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:38:31,479 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:32,103 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:33,114 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:33,479 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:34,113 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:35,120 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:35,240 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:38:35,240 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:35,241 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:35,352 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 22:38:35,352 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 22:38:35,480 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:38:35,480 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:38:35,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 22:38:35,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media +2024-12-31 22:38:35,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table/eval +2024-12-31 22:38:35,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table +2024-12-31 22:38:35,564 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:38:35,664 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/c8l6zeit-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 22:38:35,804 INFO wandb-upload_1:1759480 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpvhsttq9_ +2024-12-31 22:38:35,976 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 22:38:35,987 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:38:36,104 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:36,235 INFO SenderThread:1759480 [sender.py:send_request_log_artifact():1525] logged artifact run-3bbvaqak-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE2NzU1NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MzkwNQ==', 'latestArtifact': None}} +2024-12-31 22:38:36,235 DEBUG SenderThread:1759480 [sender.py:send():379] send: files +2024-12-31 22:38:36,235 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2024-12-31 22:38:36,235 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:38:36,235 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,236 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,236 DEBUG SenderThread:1759480 [sender.py:send():379] send: telemetry +2024-12-31 22:38:36,236 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,239 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,240 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,240 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,240 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,240 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,241 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,241 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,241 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,241 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,242 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,242 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,242 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,242 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,243 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,243 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,243 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,245 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,246 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,247 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,247 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,247 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,247 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 22:38:36,480 INFO wandb-upload_2:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/dxc497fk-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 22:38:36,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:38:36,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:38:36,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table/eval/total_success_rate_ci_6_6372d751b56b6efde046.table.json +2024-12-31 22:38:36,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table/eval +2024-12-31 22:38:36,603 INFO wandb-upload_1:1759480 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpqm7wnaml +2024-12-31 22:38:36,678 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/o25kszuh-media/table/eval/total_success_rate_ci_6_6372d751b56b6efde046.table.json +2024-12-31 22:38:37,047 INFO SenderThread:1759480 [sender.py:send_request_log_artifact():1525] logged artifact run-3bbvaqak-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE2NzU2OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MzkwOA==', 'latestArtifact': None}} +2024-12-31 22:38:37,047 DEBUG SenderThread:1759480 [sender.py:send():379] send: files +2024-12-31 22:38:37,047 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_6372d751b56b6efde046.table.json with policy now +2024-12-31 22:38:37,047 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:38:37,047 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:38:37,047 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:37,048 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:37,105 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:37,205 DEBUG SenderThread:1759480 [sender.py:send():379] send: exit +2024-12-31 22:38:37,205 INFO SenderThread:1759480 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 22:38:37,206 INFO SenderThread:1759480 [sender.py:send_exit():588] handling runtime: 284 +2024-12-31 22:38:37,206 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:37,206 INFO SenderThread:1759480 [sender.py:send_exit():594] send defer +2024-12-31 22:38:37,206 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:37,206 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 22:38:37,206 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:37,206 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 22:38:37,206 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 1 +2024-12-31 22:38:37,206 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:37,207 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 22:38:37,207 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:37,207 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 22:38:37,207 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 2 +2024-12-31 22:38:37,207 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:37,207 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 22:38:37,207 INFO HandlerThread:1759480 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 22:38:37,207 INFO HandlerThread:1759480 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 22:38:37,207 INFO HandlerThread:1759480 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 22:38:37,208 DEBUG SystemMonitor:1759480 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 22:38:37,208 DEBUG SystemMonitor:1759480 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 22:38:37,260 INFO HandlerThread:1759480 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 22:38:37,260 INFO HandlerThread:1759480 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 22:38:37,260 INFO HandlerThread:1759480 [interfaces.py:finish():200] Joined network monitor +2024-12-31 22:38:37,261 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:37,261 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 22:38:37,261 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 3 +2024-12-31 22:38:37,261 DEBUG SenderThread:1759480 [sender.py:send():379] send: stats +2024-12-31 22:38:37,261 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:37,261 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 22:38:37,261 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:37,261 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 22:38:37,261 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 4 +2024-12-31 22:38:37,261 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:37,261 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 22:38:37,325 INFO wandb-upload_2:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/33tyrnx4-media/table/eval/total_success_rate_ci_6_6372d751b56b6efde046.table.json +2024-12-31 22:38:37,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:37,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:38:39,482 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:42,483 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/config.yaml +2024-12-31 22:38:44,248 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:38:44,248 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:38:44,248 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 22:38:44,249 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:44,249 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 22:38:44,249 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 5 +2024-12-31 22:38:44,249 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:38:44,249 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:38:44,249 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:44,250 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:44,250 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 22:38:44,250 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:44,251 DEBUG SenderThread:1759480 [sender.py:send():379] send: summary +2024-12-31 22:38:44,251 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:44,251 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:44,251 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 22:38:44,251 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 6 +2024-12-31 22:38:44,251 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:44,251 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 22:38:44,251 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:44,251 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 22:38:44,252 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 7 +2024-12-31 22:38:44,252 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:38:44,252 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:44,252 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 22:38:44,252 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:44,252 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 22:38:44,484 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:38:45,207 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:38:45,309 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 8 +2024-12-31 22:38:45,309 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:38:45,309 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:45,309 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 22:38:45,310 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:45,310 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 22:38:45,310 INFO SenderThread:1759480 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 22:38:45,312 INFO SenderThread:1759480 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 22:38:45,314 INFO SenderThread:1759480 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 22:38:45,328 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 9 +2024-12-31 22:38:45,329 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:45,329 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 22:38:45,329 DEBUG SenderThread:1759480 [sender.py:send():379] send: artifact +2024-12-31 22:38:45,485 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:45,775 INFO wandb-upload_1:1759480 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1lk1rrzv +2024-12-31 22:38:45,775 INFO wandb-upload_2:1759480 [upload_job.py:push():85] Skipped uploading /tmp/tmpib2suzf2/wandb-job.json +2024-12-31 22:38:45,897 INFO wandb-upload_0:1759480 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpsrixm6e7 +2024-12-31 22:38:46,207 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:38:46,479 INFO SenderThread:1759480 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDE2NzY3NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDE2MzgxOA==', 'versionIndex': 397}}} +2024-12-31 22:38:46,480 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:46,480 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 22:38:46,480 INFO SenderThread:1759480 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 22:38:46,485 INFO SenderThread:1759480 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files +2024-12-31 22:38:46,485 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/config.yaml config.yaml +2024-12-31 22:38:46,486 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json wandb-summary.json +2024-12-31 22:38:46,486 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/requirements.txt requirements.txt +2024-12-31 22:38:46,486 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-metadata.json wandb-metadata.json +2024-12-31 22:38:46,486 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:38:46,486 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log output.log +2024-12-31 22:38:46,486 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/diff.patch diff.patch +2024-12-31 22:38:46,489 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 22:38:46,491 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/2_eval_2_69a94bb80ce32994374e.mp4 media/videos/eval/2_eval_2_69a94bb80ce32994374e.mp4 +2024-12-31 22:38:46,491 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/0_eval_0_c2ff683945f18b252db9.mp4 media/videos/eval/0_eval_0_c2ff683945f18b252db9.mp4 +2024-12-31 22:38:46,491 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/4_eval_4_72049e080103157b9708.mp4 media/videos/eval/4_eval_4_72049e080103157b9708.mp4 +2024-12-31 22:38:46,491 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/3_eval_3_e19838f640f8887956ae.mp4 media/videos/eval/3_eval_3_e19838f640f8887956ae.mp4 +2024-12-31 22:38:46,491 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/1_eval_1_b0a9c00b6ce785ea7915.mp4 media/videos/eval/1_eval_1_b0a9c00b6ce785ea7915.mp4 +2024-12-31 22:38:46,493 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table/eval/total_success_rate_ci_6_6372d751b56b6efde046.table.json media/table/eval/total_success_rate_ci_6_6372d751b56b6efde046.table.json +2024-12-31 22:38:46,493 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 22:38:46,493 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 10 +2024-12-31 22:38:46,493 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:38:46,495 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:46,495 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 22:38:46,495 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:46,495 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 22:38:46,495 INFO SenderThread:1759480 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 22:38:46,669 INFO wandb-upload_2:1759480 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:38:46,697 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/requirements.txt +2024-12-31 22:38:46,741 INFO wandb-upload_1:1759480 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/config.yaml +2024-12-31 22:38:46,744 INFO wandb-upload_4:1759480 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:46,822 INFO wandb-upload_3:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/nnco23cm-events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:38:47,023 INFO Thread-11 (_thread_body):1759480 [sender.py:transition_state():614] send defer: 11 +2024-12-31 22:38:47,023 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:47,023 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 22:38:47,023 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:47,023 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 22:38:47,023 INFO SenderThread:1759480 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 22:38:47,025 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 12 +2024-12-31 22:38:47,025 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:47,025 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 22:38:47,025 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:47,025 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 22:38:47,025 INFO SenderThread:1759480 [file_stream.py:finish():601] file stream finish called +2024-12-31 22:38:47,085 INFO SenderThread:1759480 [file_stream.py:finish():605] file stream finish is done +2024-12-31 22:38:47,086 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 13 +2024-12-31 22:38:47,086 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:47,086 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 22:38:47,086 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:47,086 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 22:38:47,086 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 14 +2024-12-31 22:38:47,086 DEBUG SenderThread:1759480 [sender.py:send():379] send: final +2024-12-31 22:38:47,086 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:47,086 DEBUG SenderThread:1759480 [sender.py:send():379] send: footer +2024-12-31 22:38:47,086 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 22:38:47,087 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:47,087 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 22:38:47,087 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:38:47,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 22:38:47,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:38:47,088 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:38:47,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 22:38:47,088 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: server_info +2024-12-31 22:38:47,089 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 22:38:47,090 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:47,141 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:38:47,141 INFO MainThread:1759480 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 22:38:47,141 INFO MainThread:1759480 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 22:38:47,142 INFO MainThread:1759480 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 22:38:47,142 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 22:38:47,142 INFO HandlerThread:1759480 [handler.py:finish():882] shutting down handler +2024-12-31 22:38:48,088 INFO WriterThread:1759480 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/run-3bbvaqak.wandb +2024-12-31 22:38:48,141 INFO SenderThread:1759480 [sender.py:finish():1615] shutting down sender +2024-12-31 22:38:48,141 INFO SenderThread:1759480 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 22:38:48,141 INFO SenderThread:1759480 [file_pusher.py:join():175] waiting for file pusher diff --git a/3bbvaqak/wandb/debug.log b/3bbvaqak/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..dca3e56e9f130731f1e44f5007217f578094e995 --- /dev/null +++ b/3bbvaqak/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_setup.py:_flush():76] Configure stats pid to 1759289 +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/settings +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/logs/debug.log +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/logs/debug-internal.log +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_init.py:init():569] calling init triggers +2024-12-31 22:33:52,164 INFO MainThread:1759289 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak'} +2024-12-31 22:33:52,164 INFO MainThread:1759289 [wandb_init.py:init():619] starting backend +2024-12-31 22:33:52,164 INFO MainThread:1759289 [wandb_init.py:init():623] setting up manager +2024-12-31 22:33:52,168 INFO MainThread:1759289 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 22:33:52,170 INFO MainThread:1759289 [wandb_init.py:init():631] backend started and connected +2024-12-31 22:33:52,182 INFO MainThread:1759289 [wandb_init.py:init():720] updated telemetry +2024-12-31 22:33:52,188 INFO MainThread:1759289 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 22:33:52,402 INFO MainThread:1759289 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 22:33:52,472 INFO MainThread:1759289 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 22:33:52,473 INFO MainThread:1759289 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 22:33:52,801 INFO MainThread:1759289 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 22:33:52,801 INFO MainThread:1759289 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 22:33:52,801 INFO MainThread:1759289 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 22:33:52,801 INFO MainThread:1759289 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 22:33:52,803 INFO MainThread:1759289 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 22:33:52,803 INFO MainThread:1759289 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/tb, True +2024-12-31 22:33:56,746 INFO MainThread:1759289 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-31 22:38:48,142 WARNING MsgRouterThr:1759289 [router.py:message_loop():77] message_loop has been closed diff --git a/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/code/FISH/eval_robot.py b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..6f971c04f88c3f951a2321c29af50f00efe5b3a1 --- /dev/null +++ b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + total_num_episodes += filtered_run.summary_metrics['episode'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/config.yaml b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd6e8860c0b883ecdef472ce8f307858e0f89568 --- /dev/null +++ b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1745_0' +wandb_notes: + desc: null + value: 1745_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735702432 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/diff.patch b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..7e3c8eb47f00846235f20cbe3844fb395661a044 --- /dev/null +++ b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/diff.patch @@ -0,0 +1,69 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..57a851f 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1007_0' ++# wandb_run_id: '1002_0' + ++# all books, 6/20 demos per book ++# RGBD+mask+act history ++# wandb_run_id: '1007_0' ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..6f971c0 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): diff --git a/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table/eval/total_success_rate_ci_6_6372d751b56b6efde046.table.json b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table/eval/total_success_rate_ci_6_6372d751b56b6efde046.table.json new file mode 100644 index 0000000000000000000000000000000000000000..10f60b0b45b6db6e6b38845cbbf4ccd7461679b5 --- /dev/null +++ b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table/eval/total_success_rate_ci_6_6372d751b56b6efde046.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[5, 20, 0.25, 0.0748827756301437, 0.5155800878970457, 0.10450506218237048, 0.4889568835518989, 0.1234313175835344, 0.42410130800334667, 0.1327295862872812, 0.4096604734839875]]} \ No newline at end of file diff --git a/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/0_eval_0_c2ff683945f18b252db9.mp4 b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/0_eval_0_c2ff683945f18b252db9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ac4c9c5a3da65c3d470071feeca0ddabdb6a7fc9 --- /dev/null +++ b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/0_eval_0_c2ff683945f18b252db9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2ff683945f18b252db9b63ccadbd7f6c106e36fe5b0ce645b8ad8a20d90c3e9 +size 1444652 diff --git a/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/1_eval_1_b0a9c00b6ce785ea7915.mp4 b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/1_eval_1_b0a9c00b6ce785ea7915.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..87e3871cd3f9ec3db4870c8f0250e762a550a85a --- /dev/null +++ b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/1_eval_1_b0a9c00b6ce785ea7915.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0a9c00b6ce785ea7915153a57e3c3a18351caca9074eb5410bdfcd1d23fb9ce +size 1498557 diff --git a/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/2_eval_2_69a94bb80ce32994374e.mp4 b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/2_eval_2_69a94bb80ce32994374e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..45cb59fd864071c8cd6e3d8a01a40ae94eb142a6 --- /dev/null +++ b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/2_eval_2_69a94bb80ce32994374e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69a94bb80ce32994374e0821259069865e7ba479f99e8189a664782f3042a1f9 +size 1468981 diff --git a/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/3_eval_3_e19838f640f8887956ae.mp4 b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/3_eval_3_e19838f640f8887956ae.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..85494cc50f8ef2461da6fffb832df46732001df5 --- /dev/null +++ b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/3_eval_3_e19838f640f8887956ae.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e19838f640f8887956aea499b2d301f845eb33dc9585229779be2198bf89afa6 +size 1541308 diff --git a/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/4_eval_4_72049e080103157b9708.mp4 b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/4_eval_4_72049e080103157b9708.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..45f36f81c9fe6ec5abb73c2e21acc98efd026a19 --- /dev/null +++ b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/4_eval_4_72049e080103157b9708.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72049e080103157b9708135d69265581d5d9a1c8d4ca1e0db76e8dbac9047f0a +size 1558272 diff --git a/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..b026a03e9480748c6d14b7acac57c1dd24ab0763 --- /dev/null +++ b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log @@ -0,0 +1,271 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735702436.747494]: resetting environment +[INFO] [1735702436.752751]: cleared current plan +[INFO] [1735702436.754014]: moving to home +[INFO] [1735702437.756525]: reached home +[INFO] [1735702437.758061]: reset action history +[INFO] [1735702440.468700]: environment reset +Starting episode 0 +[INFO] [1735702440.474360]: resetting environment +[INFO] [1735702440.480738]: cleared current plan +[INFO] [1735702440.481750]: moving to home +[INFO] [1735702441.483784]: reached home +[INFO] [1735702441.485024]: reset action history +[INFO] [1735702444.194123]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735702445.962360]: resetting environment +[INFO] [1735702445.969483]: cleared current plan +[INFO] [1735702445.970484]: moving to home +[INFO] [1735702446.972594]: reached home +[INFO] [1735702446.974152]: reset action history +[INFO] [1735702449.683510]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/episode_rosbags/episode_0_2024-12-31-22-34-09.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1759289.log +started roslaunch server http://158.130.50.37:41489/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1759612] +process[print_text-2]: started with pid [1759613] +process[rosbag_record-3]: started with pid [1759614] +started bagging! +For topic gripper_width: timestamp difference is 109948873 for nearest: 1735702454120013236 - target: 1735702454010064363 at idx 67 +For topic gripper_width: timestamp difference is 117890880 for nearest: 1735702464019988344 - target: 1735702463902097464 at idx 207 +For topic gripper_width: timestamp difference is -129075138 for nearest: 1735702481453334959 - target: 1735702481582410097 at idx 249 +[INFO] [1735702484.770043]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735702485.541094]: Stored episode 1. +[INFO] [1735702485.541377]: Saving video... +[INFO] [1735702485.739676]: Video saved! +Starting episode 1 +[INFO] [1735702485.771112]: resetting environment +[INFO] [1735702485.802037]: cleared current plan +[INFO] [1735702485.804829]: moving to home +[INFO] [1735702489.321013]: reached home +[INFO] [1735702489.321439]: reset action history +[INFO] [1735702492.035171]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735702494.599578]: resetting environment +[INFO] [1735702494.606684]: cleared current plan +[INFO] [1735702494.606867]: moving to home +[INFO] [1735702495.608331]: reached home +[INFO] [1735702495.608847]: reset action history +[INFO] [1735702498.317516]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/episode_rosbags/episode_1_2024-12-31-22-34-58.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1759289.log +started roslaunch server http://158.130.50.37:36011/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1759758] +process[print_text-5]: started with pid [1759759] +process[rosbag_record-6]: started with pid [1759760] +started bagging! +For topic gripper_width: timestamp difference is 104911434 for nearest: 1735702529353320705 - target: 1735702529248409271 at idx 249 +For topic gripper_width: timestamp difference is 107899711 for nearest: 1735702533019938276 - target: 1735702532912038565 at idx 249 +[INFO] [1735702533.772021]: Storing episode... +[rosbag_record-6] killing on exit[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735702534.570763]: Stored episode 2. +[INFO] [1735702534.578568]: Saving video... +[INFO] [1735702534.789459]: Video saved! +Starting episode 2 +[INFO] [1735702534.827448]: resetting environment +[INFO] [1735702534.859475]: cleared current plan +[INFO] [1735702534.866537]: moving to home +[INFO] [1735702538.371646]: reached home +[INFO] [1735702538.377337]: reset action history +[INFO] [1735702541.167633]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735702557.467008]: resetting environment +[INFO] [1735702557.481431]: cleared current plan +[INFO] [1735702557.481644]: moving to home +[INFO] [1735702558.483804]: reached home +[INFO] [1735702558.484500]: reset action history +[INFO] [1735702561.258272]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/episode_rosbags/episode_2_2024-12-31-22-36-01.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1759289.log +started roslaunch server http://158.130.50.37:37047/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1759901] +process[print_text-8]: started with pid [1759902] +process[rosbag_record-9]: started with pid [1759903] +started bagging! +For topic gripper_width: timestamp difference is 106345401 for nearest: 1735702572619945512 - target: 1735702572513600111 at idx 141 +For topic gripper_width: timestamp difference is 110725260 for nearest: 1735702577820024586 - target: 1735702577709299326 at idx 210 +For topic gripper_width: timestamp difference is 112579405 for nearest: 1735702580153277456 - target: 1735702580040698051 at idx 235 +[INFO] [1735702597.305468]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1735702598.056242]: Stored episode 3. +[INFO] [1735702598.057344]: Saving video... +[INFO] [1735702598.271585]: Video saved! +Starting episode 3 +[INFO] [1735702598.317269]: resetting environment +[INFO] [1735702598.341782]: cleared current plan +[INFO] [1735702598.342754]: moving to home +[INFO] [1735702601.946188]: reached home +[INFO] [1735702601.951749]: reset action history +[INFO] [1735702604.735268]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735702608.774245]: resetting environment +[INFO] [1735702608.801477]: cleared current plan +[INFO] [1735702608.801802]: moving to home +[INFO] [1735702609.805660]: reached home +[INFO] [1735702609.822998]: reset action history +[INFO] [1735702612.606298]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/episode_rosbags/episode_3_2024-12-31-22-36-52.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1759289.log +started roslaunch server http://158.130.50.37:46819/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1760042] +process[print_text-11]: started with pid [1760043] +process[rosbag_record-12]: started with pid [1760044] +started bagging! +For topic gripper_width: timestamp difference is -110142584 for nearest: 1735702634253318910 - target: 1735702634363461494 at idx 249 +[INFO] [1735702650.038041]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1735702650.857211]: Stored episode 4. +[INFO] [1735702650.869614]: Saving video... +[INFO] [1735702651.095310]: Video saved! +Starting episode 4 +[INFO] [1735702651.161628]: resetting environment +[INFO] [1735702651.184451]: cleared current plan +[INFO] [1735702651.184902]: moving to home +[INFO] [1735702654.694033]: reached home +[INFO] [1735702654.703375]: reset action history +[INFO] [1735702657.496268]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735702662.664156]: resetting environment +[INFO] [1735702662.703450]: cleared current plan +[INFO] [1735702662.704612]: moving to home +[INFO] [1735702663.707391]: reached home +[INFO] [1735702663.710956]: reset action history +[INFO] [1735702666.515264]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/episode_rosbags/episode_4_2024-12-31-22-37-46.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1759289.log +started roslaunch server http://158.130.50.37:34205/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1760185] +process[print_text-14]: started with pid [1760186] +process[rosbag_record-15]: started with pid [1760187] +started bagging! +For topic gripper_width: timestamp difference is -102688587 for nearest: 1735702678953271591 - target: 1735702679055960178 at idx 161 +For topic gripper_width: timestamp difference is -203006263 for nearest: 1735702684353318696 - target: 1735702684556324959 at idx 242 +[INFO] [1735702704.665715]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1735702705.451630]: Stored episode 5. +[INFO] [1735702705.461488]: Saving video... +[INFO] [1735702705.664552]: Video saved! +[INFO] [1735702705.742440]: resetting environment +[INFO] [1735702705.785275]: cleared current plan +[INFO] [1735702705.788815]: moving to home +[INFO] [1735702709.398172]: reached home +[INFO] [1735702709.407829]: reset action history +[INFO] [1735702712.308839]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/requirements.txt b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-metadata.json b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7ebf2f7fda8db4848d15520620a02f7874d63d9d --- /dev/null +++ b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T03:33:52.497155", + "startedAt": "2025-01-01T03:33:52.161539", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3560.59375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.004, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4496.64, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 560.3900833129883 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..0688a48ef1784b30748bad8476183a590d0ec16c --- /dev/null +++ b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "c2ff683945f18b252db9b63ccadbd7f6c106e36fe5b0ce645b8ad8a20d90c3e9", "size": 1444652, "path": "media/videos/eval/0_eval_0_c2ff683945f18b252db9.mp4"}, "global_step": 665, "_timestamp": 1735702715.8472216, "_runtime": 283.6767475605011, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "b0a9c00b6ce785ea7915153a57e3c3a18351caca9074eb5410bdfcd1d23fb9ce", "size": 1498557, "path": "media/videos/eval/1_eval_1_b0a9c00b6ce785ea7915.mp4"}, "eval/num_success": 5.0, "episode": 20.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "69a94bb80ce32994374e0821259069865e7ba479f99e8189a664782f3042a1f9", "size": 1468981, "path": "media/videos/eval/2_eval_2_69a94bb80ce32994374e.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "e19838f640f8887956aea499b2d301f845eb33dc9585229779be2198bf89afa6", "size": 1541308, "path": "media/videos/eval/3_eval_3_e19838f640f8887956ae.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "72049e080103157b9708135d69265581d5d9a1c8d4ca1e0db76e8dbac9047f0a", "size": 1558272, "path": "media/videos/eval/4_eval_4_72049e080103157b9708.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://81l9fmjo3ebxlrdswabrd2ftlkwnlon5x4e6gi4okuslddy40gll4x0stk9grzxmpkumxome9ne7qyuw3satabl174ms8eejhn42cuvgdsyrxsmqo07q2im8sr3exbif/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://d0vr1fb2zsxu0kuka6ddy8betkukqeywyesotl225h4ylbvav7315and5lj3lw7of62ugm54yy0ontx3mulmm6881inycvjbjil3grlqfur1o4seduh7zdn00odv61ty:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 5, "total_num_episodes": 20, "total_success_rate": 0.25, "total_success_rate_lb_0.01": 0.0748827756301437, "total_success_rate_ub_0.01": 0.5155800878970457, "total_success_rate_lb_0.025": 0.10450506218237048, "total_success_rate_ub_0.025": 0.4889568835518989, "total_success_rate_lb_0.05": 0.1234313175835344, "total_success_rate_ub_0.05": 0.42410130800334667, "total_success_rate_lb_0.1": 0.1327295862872812, "total_success_rate_ub_0.1": 0.4096604734839875, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "6372d751b56b6efde046fe6c6445b475d8642a5e03971bb888521185e222e0c0", "size": 507, "artifact_path": "wandb-client-artifact://r81lo76fkux393mnqnho9t65neslvpx40ea92rl020xsw29v9j475xiw4llsf4bk17s09igo90o7kuvnaxbrcg2t82g148xhr6jndul3g5l16tzvwqs3hqnjsa67mnql/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://euvyf5l7s1ppnoubmmhcir6k25tgj3e55aea1ds9yenmw4i1fwtbmtwa4ire26q9qiw6fxce7kbpqejn401lapip7a3sqdtvoh40emto7n5k92o0cdeb53mlpt09uq3h:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_6372d751b56b6efde046.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 284}, "eval/total_success_rate": 0.25, "eval/total_success_rate_lb_0.01": 0.07488277554512024, "eval/total_success_rate_ub_0.01": 0.5155800580978394, "eval/total_success_rate_lb_0.025": 0.10450506210327148, "eval/total_success_rate_ub_0.025": 0.48895689845085144, "eval/total_success_rate_lb_0.05": 0.12343131750822067, "eval/total_success_rate_ub_0.05": 0.424101322889328, "eval/total_success_rate_lb_0.1": 0.13272958993911743, "eval/total_success_rate_ub_0.1": 0.4096604883670807} \ No newline at end of file diff --git a/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/logs/debug-internal.log b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..17cabfc6d32757766e7630f77278420ac474b6ac --- /dev/null +++ b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/logs/debug-internal.log @@ -0,0 +1,801 @@ +2024-12-31 22:33:52,169 INFO StreamThr :1759480 [internal.py:wandb_internal():85] W&B internal server running at pid: 1759480, started at: 2024-12-31 22:33:52.168929 +2024-12-31 22:33:52,170 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status +2024-12-31 22:33:52,183 INFO WriterThread:1759480 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/run-3bbvaqak.wandb +2024-12-31 22:33:52,183 DEBUG SenderThread:1759480 [sender.py:send():379] send: header +2024-12-31 22:33:52,190 DEBUG SenderThread:1759480 [sender.py:send():379] send: run +2024-12-31 22:33:52,386 INFO SenderThread:1759480 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files +2024-12-31 22:33:52,386 INFO SenderThread:1759480 [sender.py:_start_run_threads():1188] run started: 3bbvaqak with start time 1735702432.170474 +2024-12-31 22:33:52,402 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 22:33:52,402 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: check_version +2024-12-31 22:33:52,479 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 22:33:52,484 DEBUG HandlerThread:1759480 [system_info.py:__init__():26] System info init +2024-12-31 22:33:52,484 DEBUG HandlerThread:1759480 [system_info.py:__init__():41] System info init done +2024-12-31 22:33:52,484 INFO HandlerThread:1759480 [system_monitor.py:start():194] Starting system monitor +2024-12-31 22:33:52,484 INFO SystemMonitor:1759480 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 22:33:52,484 INFO HandlerThread:1759480 [system_monitor.py:probe():214] Collecting system info +2024-12-31 22:33:52,485 INFO SystemMonitor:1759480 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 22:33:52,485 INFO SystemMonitor:1759480 [interfaces.py:start():188] Started disk monitoring +2024-12-31 22:33:52,486 INFO SystemMonitor:1759480 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 22:33:52,487 INFO SystemMonitor:1759480 [interfaces.py:start():188] Started memory monitoring +2024-12-31 22:33:52,487 INFO SystemMonitor:1759480 [interfaces.py:start():188] Started network monitoring +2024-12-31 22:33:52,497 DEBUG HandlerThread:1759480 [system_info.py:probe():152] Probing system +2024-12-31 22:33:52,498 DEBUG HandlerThread:1759480 [system_info.py:_probe_git():137] Probing git +2024-12-31 22:33:52,503 DEBUG HandlerThread:1759480 [system_info.py:_probe_git():145] Probing git done +2024-12-31 22:33:52,503 DEBUG HandlerThread:1759480 [system_info.py:probe():200] Probing system done +2024-12-31 22:33:52,503 DEBUG HandlerThread:1759480 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T03:33:52.497155', 'startedAt': '2025-01-01T03:33:52.161539', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3560.59375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.004, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.64, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 560.3900833129883}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 22:33:52,503 INFO HandlerThread:1759480 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 22:33:52,503 INFO HandlerThread:1759480 [system_monitor.py:probe():227] Publishing system info +2024-12-31 22:33:52,503 DEBUG HandlerThread:1759480 [system_info.py:_save_code():44] Saving code +2024-12-31 22:33:52,505 DEBUG HandlerThread:1759480 [system_info.py:_save_code():67] Saving code done +2024-12-31 22:33:52,505 DEBUG HandlerThread:1759480 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 22:33:52,567 DEBUG HandlerThread:1759480 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 22:33:52,567 INFO HandlerThread:1759480 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 22:33:52,570 DEBUG SenderThread:1759480 [sender.py:send():379] send: files +2024-12-31 22:33:52,570 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 22:33:52,570 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 22:33:52,570 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 22:33:52,800 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 22:33:52,801 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: python_packages +2024-12-31 22:33:52,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:33:52,802 DEBUG SenderThread:1759480 [sender.py:send():379] send: telemetry +2024-12-31 22:33:52,820 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/g166iolm-wandb-metadata.json +2024-12-31 22:33:52,831 INFO wandb-upload_2:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/iz96odku-diff.patch +2024-12-31 22:33:52,849 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:33:52,849 INFO HandlerThread:1759480 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/tb" + save: true +} +control { + relay_id: "bf3d693ffd2549f08dea2c0f264a209a" +} +_info { + stream_id: "3bbvaqak" +} + +2024-12-31 22:33:52,849 DEBUG HandlerThread:1759480 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 22:33:52,850 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:33:52,854 INFO wandb-upload_1:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/z57m9fzd-code/FISH/eval_robot.py +2024-12-31 22:33:53,004 DEBUG SenderThread:1759480 [sender.py:send():379] send: tbrecord +2024-12-31 22:33:53,004 DEBUG SenderThread:1759480 [sender.py:send():379] send: files +2024-12-31 22:33:53,004 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 with policy live +2024-12-31 22:33:53,188 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/dm0fawzm-events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:33:53,388 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/diff.patch +2024-12-31 22:33:53,388 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/code/FISH/eval_robot.py +2024-12-31 22:33:53,388 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:33:53,388 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:33:53,388 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-metadata.json +2024-12-31 22:33:53,388 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/requirements.txt +2024-12-31 22:33:53,389 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/code/FISH +2024-12-31 22:33:53,389 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/code +2024-12-31 22:33:53,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:33:54,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:33:55,387 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:33:55,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:33:56,746 DEBUG SenderThread:1759480 [sender.py:send():379] send: config +2024-12-31 22:33:56,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:33:57,757 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:33:57,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:33:58,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:33:59,388 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:33:59,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:00,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:01,389 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:01,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:02,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:02,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:03,389 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:03,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:04,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:05,390 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:05,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:06,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:07,391 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:07,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:34:07,801 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:34:07,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:07,910 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:08,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:09,391 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:09,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:10,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:11,392 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:11,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:12,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:13,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:13,804 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:14,804 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:15,394 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:15,804 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:16,804 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:17,804 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:18,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:18,804 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:19,804 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:20,805 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:21,805 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:22,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:34:22,801 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:34:22,845 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:23,805 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:23,922 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:24,396 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/config.yaml +2024-12-31 22:34:24,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:25,397 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:25,805 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:26,805 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:27,805 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:28,805 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:29,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:29,805 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:30,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:31,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:32,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:33,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:34,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:34,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:35,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:36,807 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:37,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:34:37,801 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:34:37,845 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:38,808 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:39,808 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:39,872 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:40,808 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:41,808 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:42,808 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:43,401 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:43,808 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:44,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:45,541 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:45,769 DEBUG SenderThread:1759480 [sender.py:send():379] send: files +2024-12-31 22:34:45,769 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c2ff683945f18b252db9.mp4 with policy now +2024-12-31 22:34:45,769 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:34:45,771 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:34:45,771 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:34:45,771 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:34:45,808 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:46,013 INFO wandb-upload_1:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/holiaaxl-media/videos/eval/0_eval_0_c2ff683945f18b252db9.mp4 +2024-12-31 22:34:46,402 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:34:46,403 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/0_eval_0_c2ff683945f18b252db9.mp4 +2024-12-31 22:34:46,403 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media +2024-12-31 22:34:46,403 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos +2024-12-31 22:34:46,403 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval +2024-12-31 22:34:46,683 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/9qjwdrvb-media/videos/eval/0_eval_0_c2ff683945f18b252db9.mp4 +2024-12-31 22:34:46,808 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:47,403 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:47,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:48,809 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:49,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:50,811 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:51,365 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:51,404 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:51,811 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:52,487 DEBUG SystemMonitor:1759480 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 22:34:52,488 DEBUG SenderThread:1759480 [sender.py:send():379] send: stats +2024-12-31 22:34:52,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:34:52,801 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:34:52,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:53,405 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:53,811 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:54,811 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:55,405 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:34:55,699 INFO wandb-upload_1:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/d371h7p7-events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:34:55,811 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:56,653 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:34:56,811 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:57,406 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:57,811 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:58,811 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:34:59,407 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:34:59,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:00,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:01,408 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:35:01,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:02,262 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:02,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:03,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:04,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:05,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:06,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:07,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:35:07,801 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:35:07,845 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:07,862 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:08,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:09,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:10,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:11,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:12,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:13,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:13,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:14,814 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:15,814 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:16,814 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:17,814 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:18,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:18,814 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:19,814 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:20,814 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:21,814 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:22,488 DEBUG SenderThread:1759480 [sender.py:send():379] send: stats +2024-12-31 22:35:22,801 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:35:22,801 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:35:22,849 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:23,815 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:23,880 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:24,815 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:25,815 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:26,816 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:27,816 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:28,816 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:29,363 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:29,816 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:30,816 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:31,417 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:35:31,816 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:32,817 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:33,817 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:34,572 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:34,820 DEBUG SenderThread:1759480 [sender.py:send():379] send: files +2024-12-31 22:35:34,820 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b0a9c00b6ce785ea7915.mp4 with policy now +2024-12-31 22:35:34,823 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:35:34,824 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:35:34,824 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:35:34,827 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:35:34,828 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:35,064 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/kofetkye-media/videos/eval/1_eval_1_b0a9c00b6ce785ea7915.mp4 +2024-12-31 22:35:35,418 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:35:35,418 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:35:35,418 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/1_eval_1_b0a9c00b6ce785ea7915.mp4 +2024-12-31 22:35:35,419 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval +2024-12-31 22:35:35,709 INFO wandb-upload_1:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/j629ah60-media/videos/eval/1_eval_1_b0a9c00b6ce785ea7915.mp4 +2024-12-31 22:35:35,829 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:36,830 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:37,419 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:35:37,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:35:37,812 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:35:37,853 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:38,832 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:39,420 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:35:39,827 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:40,413 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:40,824 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:41,832 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:42,829 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:43,421 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:35:43,828 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:44,828 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:45,812 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:45,828 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:46,831 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:47,839 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:48,839 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:49,834 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:50,815 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:50,830 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:51,834 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:52,489 DEBUG SenderThread:1759480 [sender.py:send():379] send: stats +2024-12-31 22:35:52,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:35:52,804 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:35:52,849 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:53,833 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:54,837 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:55,834 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:55,897 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:35:56,836 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:57,833 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:58,265 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:35:58,265 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:35:58,265 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:35:58,426 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:35:58,426 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:35:58,679 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/h3wua1zg-events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:35:58,838 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:35:59,426 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:35:59,842 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:00,842 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:01,261 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:01,841 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:02,840 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:03,428 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:03,841 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:05,035 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:05,428 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:06,027 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:06,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:07,027 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:07,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:36:07,802 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:36:08,027 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:09,027 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:10,027 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:11,027 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:11,890 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:12,027 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:13,028 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:13,432 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:14,028 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:15,028 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:16,028 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:17,028 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:17,805 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:18,028 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:19,028 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:19,434 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:20,029 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:21,029 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:21,434 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:22,030 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:22,489 DEBUG SenderThread:1759480 [sender.py:send():379] send: stats +2024-12-31 22:36:22,802 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:36:22,802 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:36:22,888 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:23,030 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:24,030 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:25,030 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:26,031 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:27,030 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:28,030 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:28,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:29,031 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:30,031 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:31,031 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:32,031 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:33,031 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:33,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:34,032 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:35,031 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:36,032 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:37,032 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:37,817 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:36:37,817 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:36:38,035 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:38,308 DEBUG SenderThread:1759480 [sender.py:send():379] send: files +2024-12-31 22:36:38,309 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_69a94bb80ce32994374e.mp4 with policy now +2024-12-31 22:36:38,313 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:36:38,314 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:36:38,314 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:36:38,314 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:36:38,440 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:36:38,440 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/2_eval_2_69a94bb80ce32994374e.mp4 +2024-12-31 22:36:38,440 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval +2024-12-31 22:36:38,547 INFO wandb-upload_1:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/d9zzwm60-media/videos/eval/2_eval_2_69a94bb80ce32994374e.mp4 +2024-12-31 22:36:38,827 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/s3pgh7ip-media/videos/eval/2_eval_2_69a94bb80ce32994374e.mp4 +2024-12-31 22:36:39,045 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:39,343 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:39,440 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:40,047 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:41,045 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:42,046 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:43,046 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:43,441 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:44,047 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:44,737 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:45,046 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:46,048 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:47,056 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:47,443 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:48,057 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:49,050 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:49,111 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:36:49,111 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:36:49,111 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:36:49,444 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:36:49,444 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:36:49,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:50,058 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:51,057 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:51,444 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:52,070 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:52,490 DEBUG SenderThread:1759480 [sender.py:send():379] send: stats +2024-12-31 22:36:52,806 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:36:52,807 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:36:53,063 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:53,445 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:54,067 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:55,059 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:55,321 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:36:55,446 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:56,062 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:57,060 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:57,446 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:36:58,060 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:36:59,061 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:00,061 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:00,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:01,061 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:02,061 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:03,062 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:04,062 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:05,062 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:05,813 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:06,063 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:07,063 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:07,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:37:07,803 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:37:08,063 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:09,063 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:10,063 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:10,920 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:11,063 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:12,063 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:13,064 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:14,064 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:15,064 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:15,453 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:37:16,064 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:16,494 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:17,064 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:18,066 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:19,064 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:20,065 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:21,065 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:21,815 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:22,065 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:22,490 DEBUG SenderThread:1759480 [sender.py:send():379] send: stats +2024-12-31 22:37:22,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:37:22,803 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:37:23,065 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:24,065 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:25,065 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:26,065 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:26,914 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:27,065 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:28,065 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:29,066 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:30,081 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:31,076 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:31,148 DEBUG SenderThread:1759480 [sender.py:send():379] send: files +2024-12-31 22:37:31,149 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_e19838f640f8887956ae.mp4 with policy now +2024-12-31 22:37:31,149 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:37:31,150 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:37:31,150 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:37:31,158 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:37:31,389 INFO wandb-upload_1:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/ns4oe1jc-media/videos/eval/3_eval_3_e19838f640f8887956ae.mp4 +2024-12-31 22:37:31,458 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:37:31,458 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:37:31,459 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/3_eval_3_e19838f640f8887956ae.mp4 +2024-12-31 22:37:31,459 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval +2024-12-31 22:37:31,745 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/mn74s48n-media/videos/eval/3_eval_3_e19838f640f8887956ae.mp4 +2024-12-31 22:37:32,071 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:32,185 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:33,076 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:33,459 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:37:34,080 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:35,078 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:35,460 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:37:36,075 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:37,076 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:37,497 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:37,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:37:37,803 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:37:38,077 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:39,078 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:39,461 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:37:40,087 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:41,094 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:42,087 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:42,617 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:43,084 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:43,174 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:37:43,175 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:37:43,175 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:37:43,462 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:37:43,462 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:37:43,463 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:37:43,705 INFO wandb-upload_1:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/mdfu3xib-events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:37:44,086 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:45,087 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:45,463 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:37:46,089 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:47,090 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:47,464 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:37:47,816 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:48,087 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:49,085 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:50,086 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:51,085 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:51,465 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:37:52,085 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:52,491 DEBUG SenderThread:1759480 [sender.py:send():379] send: stats +2024-12-31 22:37:52,803 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:37:52,804 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:37:52,934 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:53,087 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:54,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:55,087 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:56,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:57,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:58,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:37:58,815 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:37:59,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:00,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:01,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:01,468 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:02,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:03,089 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:03,815 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:38:04,089 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:05,089 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:05,470 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:06,089 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:07,089 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:07,804 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:38:07,804 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:38:08,090 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:08,907 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:38:09,090 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:10,090 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:11,090 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:12,090 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:13,090 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:14,091 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:14,815 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:38:15,091 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:16,091 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:17,091 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:18,091 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:19,091 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:19,815 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:38:20,092 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:21,091 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:22,094 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:22,491 DEBUG SenderThread:1759480 [sender.py:send():379] send: stats +2024-12-31 22:38:22,804 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 22:38:22,804 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: stop_status +2024-12-31 22:38:23,092 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:24,092 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:25,095 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:25,453 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:38:25,477 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:25,734 DEBUG SenderThread:1759480 [sender.py:send():379] send: files +2024-12-31 22:38:25,734 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_72049e080103157b9708.mp4 with policy now +2024-12-31 22:38:25,735 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:38:25,735 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:38:25,735 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:25,737 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:25,978 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/neu09czi-media/videos/eval/4_eval_4_72049e080103157b9708.mp4 +2024-12-31 22:38:26,095 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:26,477 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:38:26,477 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/4_eval_4_72049e080103157b9708.mp4 +2024-12-31 22:38:26,477 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval +2024-12-31 22:38:26,799 INFO wandb-upload_1:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/5yizbm4v-media/videos/eval/4_eval_4_72049e080103157b9708.mp4 +2024-12-31 22:38:27,123 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:27,477 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:28,102 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:29,116 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:30,106 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:31,118 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:31,445 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:38:31,479 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:32,103 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:33,114 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:33,479 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:34,113 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:35,120 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:35,240 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:38:35,240 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:35,241 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:35,352 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 22:38:35,352 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 22:38:35,480 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:38:35,480 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:38:35,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 22:38:35,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media +2024-12-31 22:38:35,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table/eval +2024-12-31 22:38:35,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table +2024-12-31 22:38:35,564 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:38:35,664 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/c8l6zeit-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 22:38:35,804 INFO wandb-upload_1:1759480 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpvhsttq9_ +2024-12-31 22:38:35,976 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 22:38:35,987 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 22:38:36,104 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:36,235 INFO SenderThread:1759480 [sender.py:send_request_log_artifact():1525] logged artifact run-3bbvaqak-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE2NzU1NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MzkwNQ==', 'latestArtifact': None}} +2024-12-31 22:38:36,235 DEBUG SenderThread:1759480 [sender.py:send():379] send: files +2024-12-31 22:38:36,235 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2024-12-31 22:38:36,235 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:38:36,235 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,236 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,236 DEBUG SenderThread:1759480 [sender.py:send():379] send: telemetry +2024-12-31 22:38:36,236 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,239 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,240 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,240 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,240 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,240 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,241 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,241 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,241 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,241 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,242 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,242 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,242 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,242 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,243 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,243 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,243 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,245 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,246 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,247 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,247 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:36,247 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:36,247 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 22:38:36,480 INFO wandb-upload_2:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/dxc497fk-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 22:38:36,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:38:36,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:38:36,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table/eval/total_success_rate_ci_6_6372d751b56b6efde046.table.json +2024-12-31 22:38:36,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table/eval +2024-12-31 22:38:36,603 INFO wandb-upload_1:1759480 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpqm7wnaml +2024-12-31 22:38:36,678 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/o25kszuh-media/table/eval/total_success_rate_ci_6_6372d751b56b6efde046.table.json +2024-12-31 22:38:37,047 INFO SenderThread:1759480 [sender.py:send_request_log_artifact():1525] logged artifact run-3bbvaqak-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE2NzU2OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MzkwOA==', 'latestArtifact': None}} +2024-12-31 22:38:37,047 DEBUG SenderThread:1759480 [sender.py:send():379] send: files +2024-12-31 22:38:37,047 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_6372d751b56b6efde046.table.json with policy now +2024-12-31 22:38:37,047 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:38:37,047 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:38:37,047 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:37,048 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:37,105 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:37,205 DEBUG SenderThread:1759480 [sender.py:send():379] send: exit +2024-12-31 22:38:37,205 INFO SenderThread:1759480 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 22:38:37,206 INFO SenderThread:1759480 [sender.py:send_exit():588] handling runtime: 284 +2024-12-31 22:38:37,206 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:37,206 INFO SenderThread:1759480 [sender.py:send_exit():594] send defer +2024-12-31 22:38:37,206 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:37,206 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 22:38:37,206 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:37,206 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 22:38:37,206 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 1 +2024-12-31 22:38:37,206 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:37,207 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 22:38:37,207 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:37,207 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 22:38:37,207 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 2 +2024-12-31 22:38:37,207 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:37,207 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 22:38:37,207 INFO HandlerThread:1759480 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 22:38:37,207 INFO HandlerThread:1759480 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 22:38:37,207 INFO HandlerThread:1759480 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 22:38:37,208 DEBUG SystemMonitor:1759480 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 22:38:37,208 DEBUG SystemMonitor:1759480 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 22:38:37,260 INFO HandlerThread:1759480 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 22:38:37,260 INFO HandlerThread:1759480 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 22:38:37,260 INFO HandlerThread:1759480 [interfaces.py:finish():200] Joined network monitor +2024-12-31 22:38:37,261 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:37,261 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 22:38:37,261 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 3 +2024-12-31 22:38:37,261 DEBUG SenderThread:1759480 [sender.py:send():379] send: stats +2024-12-31 22:38:37,261 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:37,261 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 22:38:37,261 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:37,261 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 22:38:37,261 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 4 +2024-12-31 22:38:37,261 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:37,261 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 22:38:37,325 INFO wandb-upload_2:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/33tyrnx4-media/table/eval/total_success_rate_ci_6_6372d751b56b6efde046.table.json +2024-12-31 22:38:37,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:37,481 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:38:39,482 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:42,483 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/config.yaml +2024-12-31 22:38:44,248 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:38:44,248 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:38:44,248 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 22:38:44,249 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:44,249 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 22:38:44,249 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 5 +2024-12-31 22:38:44,249 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:38:44,249 DEBUG SenderThread:1759480 [sender.py:send():379] send: history +2024-12-31 22:38:44,249 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: summary_record +2024-12-31 22:38:44,250 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:44,250 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 22:38:44,250 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:44,251 DEBUG SenderThread:1759480 [sender.py:send():379] send: summary +2024-12-31 22:38:44,251 INFO SenderThread:1759480 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 22:38:44,251 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:44,251 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 22:38:44,251 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 6 +2024-12-31 22:38:44,251 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:44,251 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 22:38:44,251 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:44,251 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 22:38:44,252 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 7 +2024-12-31 22:38:44,252 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 22:38:44,252 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:44,252 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 22:38:44,252 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:44,252 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 22:38:44,484 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:38:45,207 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:38:45,309 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 8 +2024-12-31 22:38:45,309 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:38:45,309 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:45,309 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 22:38:45,310 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:45,310 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 22:38:45,310 INFO SenderThread:1759480 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 22:38:45,312 INFO SenderThread:1759480 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 22:38:45,314 INFO SenderThread:1759480 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 22:38:45,328 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 9 +2024-12-31 22:38:45,329 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:45,329 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 22:38:45,329 DEBUG SenderThread:1759480 [sender.py:send():379] send: artifact +2024-12-31 22:38:45,485 INFO Thread-12 :1759480 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:45,775 INFO wandb-upload_1:1759480 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1lk1rrzv +2024-12-31 22:38:45,775 INFO wandb-upload_2:1759480 [upload_job.py:push():85] Skipped uploading /tmp/tmpib2suzf2/wandb-job.json +2024-12-31 22:38:45,897 INFO wandb-upload_0:1759480 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpsrixm6e7 +2024-12-31 22:38:46,207 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:38:46,479 INFO SenderThread:1759480 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDE2NzY3NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDE2MzgxOA==', 'versionIndex': 397}}} +2024-12-31 22:38:46,480 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:46,480 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 22:38:46,480 INFO SenderThread:1759480 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 22:38:46,485 INFO SenderThread:1759480 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files +2024-12-31 22:38:46,485 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/config.yaml config.yaml +2024-12-31 22:38:46,486 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json wandb-summary.json +2024-12-31 22:38:46,486 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/requirements.txt requirements.txt +2024-12-31 22:38:46,486 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-metadata.json wandb-metadata.json +2024-12-31 22:38:46,486 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:38:46,486 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log output.log +2024-12-31 22:38:46,486 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/diff.patch diff.patch +2024-12-31 22:38:46,489 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 22:38:46,491 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/2_eval_2_69a94bb80ce32994374e.mp4 media/videos/eval/2_eval_2_69a94bb80ce32994374e.mp4 +2024-12-31 22:38:46,491 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/0_eval_0_c2ff683945f18b252db9.mp4 media/videos/eval/0_eval_0_c2ff683945f18b252db9.mp4 +2024-12-31 22:38:46,491 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/4_eval_4_72049e080103157b9708.mp4 media/videos/eval/4_eval_4_72049e080103157b9708.mp4 +2024-12-31 22:38:46,491 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/3_eval_3_e19838f640f8887956ae.mp4 media/videos/eval/3_eval_3_e19838f640f8887956ae.mp4 +2024-12-31 22:38:46,491 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/videos/eval/1_eval_1_b0a9c00b6ce785ea7915.mp4 media/videos/eval/1_eval_1_b0a9c00b6ce785ea7915.mp4 +2024-12-31 22:38:46,493 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table/eval/total_success_rate_ci_6_6372d751b56b6efde046.table.json media/table/eval/total_success_rate_ci_6_6372d751b56b6efde046.table.json +2024-12-31 22:38:46,493 INFO SenderThread:1759480 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2024-12-31 22:38:46,493 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 10 +2024-12-31 22:38:46,493 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:38:46,495 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:46,495 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 22:38:46,495 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:46,495 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 22:38:46,495 INFO SenderThread:1759480 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 22:38:46,669 INFO wandb-upload_2:1759480 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/wandb-summary.json +2024-12-31 22:38:46,697 INFO wandb-upload_0:1759480 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/requirements.txt +2024-12-31 22:38:46,741 INFO wandb-upload_1:1759480 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/config.yaml +2024-12-31 22:38:46,744 INFO wandb-upload_4:1759480 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/files/output.log +2024-12-31 22:38:46,822 INFO wandb-upload_3:1759480 [upload_job.py:push():130] Uploaded file /tmp/tmpn9kjcegrwandb/nnco23cm-events.out.tfevents.1735702432.leonmkim-ROG-Strix-G15CS-G15CS.1759289.0 +2024-12-31 22:38:47,023 INFO Thread-11 (_thread_body):1759480 [sender.py:transition_state():614] send defer: 11 +2024-12-31 22:38:47,023 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:47,023 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 22:38:47,023 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:47,023 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 22:38:47,023 INFO SenderThread:1759480 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 22:38:47,025 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 12 +2024-12-31 22:38:47,025 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:47,025 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 22:38:47,025 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:47,025 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 22:38:47,025 INFO SenderThread:1759480 [file_stream.py:finish():601] file stream finish called +2024-12-31 22:38:47,085 INFO SenderThread:1759480 [file_stream.py:finish():605] file stream finish is done +2024-12-31 22:38:47,086 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 13 +2024-12-31 22:38:47,086 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:47,086 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 22:38:47,086 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:47,086 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 22:38:47,086 INFO SenderThread:1759480 [sender.py:transition_state():614] send defer: 14 +2024-12-31 22:38:47,086 DEBUG SenderThread:1759480 [sender.py:send():379] send: final +2024-12-31 22:38:47,086 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: defer +2024-12-31 22:38:47,086 DEBUG SenderThread:1759480 [sender.py:send():379] send: footer +2024-12-31 22:38:47,086 INFO HandlerThread:1759480 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 22:38:47,087 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: defer +2024-12-31 22:38:47,087 INFO SenderThread:1759480 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 22:38:47,087 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:38:47,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 22:38:47,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 22:38:47,088 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:38:47,088 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 22:38:47,088 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: server_info +2024-12-31 22:38:47,089 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 22:38:47,090 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 22:38:47,141 DEBUG SenderThread:1759480 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 22:38:47,141 INFO MainThread:1759480 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 22:38:47,141 INFO MainThread:1759480 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 22:38:47,142 INFO MainThread:1759480 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 22:38:47,142 DEBUG HandlerThread:1759480 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 22:38:47,142 INFO HandlerThread:1759480 [handler.py:finish():882] shutting down handler +2024-12-31 22:38:48,088 INFO WriterThread:1759480 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/run-3bbvaqak.wandb +2024-12-31 22:38:48,141 INFO SenderThread:1759480 [sender.py:finish():1615] shutting down sender +2024-12-31 22:38:48,141 INFO SenderThread:1759480 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 22:38:48,141 INFO SenderThread:1759480 [file_pusher.py:join():175] waiting for file pusher diff --git a/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/logs/debug.log b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..dca3e56e9f130731f1e44f5007217f578094e995 --- /dev/null +++ b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_setup.py:_flush():76] Configure stats pid to 1759289 +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/settings +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/logs/debug.log +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/logs/debug-internal.log +2024-12-31 22:33:52,163 INFO MainThread:1759289 [wandb_init.py:init():569] calling init triggers +2024-12-31 22:33:52,164 INFO MainThread:1759289 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak'} +2024-12-31 22:33:52,164 INFO MainThread:1759289 [wandb_init.py:init():619] starting backend +2024-12-31 22:33:52,164 INFO MainThread:1759289 [wandb_init.py:init():623] setting up manager +2024-12-31 22:33:52,168 INFO MainThread:1759289 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 22:33:52,170 INFO MainThread:1759289 [wandb_init.py:init():631] backend started and connected +2024-12-31 22:33:52,182 INFO MainThread:1759289 [wandb_init.py:init():720] updated telemetry +2024-12-31 22:33:52,188 INFO MainThread:1759289 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 22:33:52,402 INFO MainThread:1759289 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 22:33:52,472 INFO MainThread:1759289 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 22:33:52,473 INFO MainThread:1759289 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 22:33:52,801 INFO MainThread:1759289 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 22:33:52,801 INFO MainThread:1759289 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 22:33:52,801 INFO MainThread:1759289 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 22:33:52,801 INFO MainThread:1759289 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 22:33:52,803 INFO MainThread:1759289 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 22:33:52,803 INFO MainThread:1759289 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/3bbvaqak/tb, True +2024-12-31 22:33:56,746 INFO MainThread:1759289 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-31 22:38:48,142 WARNING MsgRouterThr:1759289 [router.py:message_loop():77] message_loop has been closed diff --git a/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/run-3bbvaqak.wandb b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/run-3bbvaqak.wandb new file mode 100644 index 0000000000000000000000000000000000000000..45e88a4c94fb923892b209a5f934ba62c4cd48eb Binary files /dev/null and b/3bbvaqak/wandb/run-20241231_223352-3bbvaqak/run-3bbvaqak.wandb differ diff --git a/71icjjya/.hydra/config.yaml b/71icjjya/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57f0faa93e8c5310ed098ffd123244267c23fcf8 --- /dev/null +++ b/71icjjya/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1745_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/71icjjya/.hydra/hydra.yaml b/71icjjya/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8bf8e1fd14c19dbb8ceeb217d6472d8e70f2a36a --- /dev/null +++ b/71icjjya/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/71icjjya/.hydra/overrides.yaml b/71icjjya/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/71icjjya/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/71icjjya/episode_rosbags/aligned_depth_to_color_K.npy b/71icjjya/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/71icjjya/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/71icjjya/episode_rosbags/cam_tf_world.npy b/71icjjya/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/71icjjya/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/71icjjya/episode_rosbags/color_K.npy b/71icjjya/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/71icjjya/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/71icjjya/episode_rosbags/depth_K.npy b/71icjjya/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/71icjjya/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/71icjjya/eval_robot.log b/71icjjya/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..08aeddb10ddf91b4434f8abd9ea3c927cad726ec --- /dev/null +++ b/71icjjya/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-31 23:52:26,718][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-31 23:52:26,721][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/71icjjya/eval_video/0_eval.mp4 b/71icjjya/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ecdcd457c05e288107cfae93b7e71bd330e3a165 --- /dev/null +++ b/71icjjya/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65829ddfa6c31573c9e76629a013199ca03a049a3a08c05340a44cb6852c78be +size 1464590 diff --git a/71icjjya/eval_video/1_eval.mp4 b/71icjjya/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..31cbb80452406f3b5e6bdf02060ba1f97d2c92d7 --- /dev/null +++ b/71icjjya/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45cd68318424e8a1aa34a9f5ea973589e5135101a19cfe2f08f18f61f3851619 +size 1446227 diff --git a/71icjjya/eval_video/2_eval.mp4 b/71icjjya/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..860f9081fb6ea42456494d29b4bd6aa7431670c2 --- /dev/null +++ b/71icjjya/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0389095f97a9dda10a03d471d919762d282d87196599d44d2c89accb3edba1a7 +size 997527 diff --git a/71icjjya/eval_video/3_eval.mp4 b/71icjjya/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..471fda56802c5192782ea7e6893451eab913641f --- /dev/null +++ b/71icjjya/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe95f75420ab4bf1f2e0926922851a7e4b0701b9e9046a2025e526945b5389a9 +size 1488529 diff --git a/71icjjya/eval_video/4_eval.mp4 b/71icjjya/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0b7ff5e8f424f55c9ca2dcc100976e990c8a5197 --- /dev/null +++ b/71icjjya/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c467f37c076e27eccc382a3a0643d361211b905835828e0362c46589ed733e2 +size 1464399 diff --git a/71icjjya/tb/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 b/71icjjya/tb/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 new file mode 100644 index 0000000000000000000000000000000000000000..54efe783ab653f10522cfa094955ecdc90132cad --- /dev/null +++ b/71icjjya/tb/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1739307fc8f5647ec15f437bc4327b8ef16565908aa5dc09a2321d79a42bff7a +size 6539 diff --git a/71icjjya/wandb/debug-internal.log b/71icjjya/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6b1fe655b53ae0070063943c5dddf41f9cc428ac --- /dev/null +++ b/71icjjya/wandb/debug-internal.log @@ -0,0 +1,957 @@ +2024-12-31 23:52:32,529 INFO StreamThr :1770882 [internal.py:wandb_internal():85] W&B internal server running at pid: 1770882, started at: 2024-12-31 23:52:32.528874 +2024-12-31 23:52:32,531 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status +2024-12-31 23:52:32,545 INFO WriterThread:1770882 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/run-71icjjya.wandb +2024-12-31 23:52:32,545 DEBUG SenderThread:1770882 [sender.py:send():379] send: header +2024-12-31 23:52:32,552 DEBUG SenderThread:1770882 [sender.py:send():379] send: run +2024-12-31 23:52:32,774 INFO SenderThread:1770882 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files +2024-12-31 23:52:32,774 INFO SenderThread:1770882 [sender.py:_start_run_threads():1188] run started: 71icjjya with start time 1735707152.529045 +2024-12-31 23:52:32,789 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 23:52:32,790 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: check_version +2024-12-31 23:52:32,836 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 23:52:32,844 DEBUG HandlerThread:1770882 [system_info.py:__init__():26] System info init +2024-12-31 23:52:32,844 DEBUG HandlerThread:1770882 [system_info.py:__init__():41] System info init done +2024-12-31 23:52:32,845 INFO HandlerThread:1770882 [system_monitor.py:start():194] Starting system monitor +2024-12-31 23:52:32,845 INFO SystemMonitor:1770882 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 23:52:32,845 INFO HandlerThread:1770882 [system_monitor.py:probe():214] Collecting system info +2024-12-31 23:52:32,845 INFO SystemMonitor:1770882 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 23:52:32,846 INFO SystemMonitor:1770882 [interfaces.py:start():188] Started disk monitoring +2024-12-31 23:52:32,847 INFO SystemMonitor:1770882 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 23:52:32,847 INFO SystemMonitor:1770882 [interfaces.py:start():188] Started memory monitoring +2024-12-31 23:52:32,848 INFO SystemMonitor:1770882 [interfaces.py:start():188] Started network monitoring +2024-12-31 23:52:32,858 DEBUG HandlerThread:1770882 [system_info.py:probe():152] Probing system +2024-12-31 23:52:32,859 DEBUG HandlerThread:1770882 [system_info.py:_probe_git():137] Probing git +2024-12-31 23:52:32,864 DEBUG HandlerThread:1770882 [system_info.py:_probe_git():145] Probing git done +2024-12-31 23:52:32,864 DEBUG HandlerThread:1770882 [system_info.py:probe():200] Probing system done +2024-12-31 23:52:32,864 DEBUG HandlerThread:1770882 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T04:52:32.858202', 'startedAt': '2025-01-01T04:52:32.521194', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.6215, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.485, 'min': 800.0, 'max': 4700.0}, {'current': 4499.702, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 621.5189628601074}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 23:52:32,864 INFO HandlerThread:1770882 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 23:52:32,864 INFO HandlerThread:1770882 [system_monitor.py:probe():227] Publishing system info +2024-12-31 23:52:32,864 DEBUG HandlerThread:1770882 [system_info.py:_save_code():44] Saving code +2024-12-31 23:52:32,866 DEBUG HandlerThread:1770882 [system_info.py:_save_code():67] Saving code done +2024-12-31 23:52:32,866 DEBUG HandlerThread:1770882 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 23:52:32,927 DEBUG HandlerThread:1770882 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 23:52:32,928 INFO HandlerThread:1770882 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 23:52:32,931 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:52:32,931 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 23:52:32,932 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 23:52:32,932 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 23:52:33,166 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/5pwmmmvb-wandb-metadata.json +2024-12-31 23:52:33,167 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 23:52:33,167 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: python_packages +2024-12-31 23:52:33,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:52:33,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:33,169 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:52:33,171 INFO HandlerThread:1770882 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/tb" + save: true +} +control { + relay_id: "ed18d94dda074ba88639ea3a7fdd48f4" +} +_info { + stream_id: "71icjjya" +} + +2024-12-31 23:52:33,171 DEBUG HandlerThread:1770882 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 23:52:33,188 INFO wandb-upload_2:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/b3lguk4v-diff.patch +2024-12-31 23:52:33,223 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/vn55u65j-code/FISH/eval_robot.py +2024-12-31 23:52:33,229 DEBUG SenderThread:1770882 [sender.py:send():379] send: telemetry +2024-12-31 23:52:33,229 DEBUG SenderThread:1770882 [sender.py:send():379] send: tbrecord +2024-12-31 23:52:33,230 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:52:33,230 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 with policy live +2024-12-31 23:52:33,418 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/ioz6e7gv-events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:52:33,776 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:52:33,776 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:52:33,776 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-metadata.json +2024-12-31 23:52:33,777 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/requirements.txt +2024-12-31 23:52:33,777 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/code/FISH/eval_robot.py +2024-12-31 23:52:33,777 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/diff.patch +2024-12-31 23:52:33,777 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/code/FISH +2024-12-31 23:52:33,777 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/code +2024-12-31 23:52:34,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:35,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:35,775 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:52:36,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:37,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:38,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:38,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:39,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:40,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:41,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:42,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:43,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:43,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:44,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:45,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:45,872 DEBUG SenderThread:1770882 [sender.py:send():379] send: config +2024-12-31 23:52:46,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:47,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:47,779 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:52:48,167 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:52:48,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:52:48,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:48,235 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:49,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:50,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:51,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:52,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:53,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:53,780 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:52:54,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:54,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:55,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:56,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:57,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:57,781 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:52:58,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:59,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:59,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:59,782 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:00,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:01,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:01,783 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:02,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:03,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:53:03,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:53:03,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:04,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:04,224 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:04,784 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/config.yaml +2024-12-31 23:53:05,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:05,784 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:06,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:07,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:07,785 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:08,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:09,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:09,517 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:09,785 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:10,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:11,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:12,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:13,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:14,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:15,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:15,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:16,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:17,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:18,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:53:18,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:53:18,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:19,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:20,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:20,301 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:21,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:21,789 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:22,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:23,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:23,789 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:24,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:25,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:25,759 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:26,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:27,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:27,790 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:28,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:29,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:29,791 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:30,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:30,766 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:31,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:31,791 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:32,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:32,848 DEBUG SystemMonitor:1770882 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 23:53:32,849 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:53:33,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:53:33,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:53:33,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:34,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:35,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:36,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:36,541 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:37,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:38,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:39,176 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:40,176 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:41,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:42,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:42,176 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:43,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:44,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:45,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:46,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:47,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:48,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:53:48,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:53:48,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:48,269 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:49,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:50,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:51,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:52,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:53,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:54,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:54,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:55,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:56,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:57,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:58,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:59,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:59,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:00,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:01,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:02,179 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:02,849 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:54:03,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:54:03,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:54:03,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:03,799 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:04,179 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:04,269 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:05,185 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:05,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:54:05,262 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:54:05,262 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 with policy now +2024-12-31 23:54:05,262 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:54:05,262 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:54:05,263 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:54:05,535 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/abdxmvd5-media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 +2024-12-31 23:54:05,800 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:05,800 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 +2024-12-31 23:54:05,800 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:54:05,800 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos +2024-12-31 23:54:05,801 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval +2024-12-31 23:54:05,801 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media +2024-12-31 23:54:06,089 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/26nhcuhz-media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 +2024-12-31 23:54:06,184 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:07,183 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:07,801 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:08,183 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:09,184 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:10,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:10,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:11,185 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:11,802 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:12,185 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:13,185 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:14,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:15,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:15,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:16,185 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:17,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:17,804 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:18,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:54:18,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:54:18,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:19,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:20,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:21,101 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:21,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:21,805 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:22,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:23,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:24,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:25,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:26,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:26,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:27,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:28,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:29,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:30,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:31,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:31,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:32,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:32,810 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:54:32,850 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:54:33,049 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/s91prls9-events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:54:33,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:54:33,169 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:54:33,213 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:33,810 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:34,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:35,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:36,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:36,227 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:37,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:38,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:39,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:39,812 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:40,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:41,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:42,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:42,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:43,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:43,813 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:44,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:45,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:45,814 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:46,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:47,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:47,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:47,815 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:48,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:54:48,170 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:54:48,213 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:49,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:50,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:51,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:52,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:52,275 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:53,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:54,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:55,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:56,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:57,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:58,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:58,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:59,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:00,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:01,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:02,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:02,850 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:55:03,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:55:03,171 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:55:03,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:03,308 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:04,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:05,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:06,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:07,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:08,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:09,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:09,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:10,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:11,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:12,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:13,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:13,823 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:14,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:14,773 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:15,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:15,824 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:16,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:17,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:17,825 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:18,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:55:18,171 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:55:18,213 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:19,193 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:20,167 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:20,193 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:20,916 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:55:20,916 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 with policy now +2024-12-31 23:55:20,919 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:55:20,920 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:55:20,920 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:55:20,925 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:55:21,183 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/2citjgvy-media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 +2024-12-31 23:55:21,195 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:21,826 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:21,826 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:55:21,826 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 +2024-12-31 23:55:21,826 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval +2024-12-31 23:55:22,127 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/17gsyab6-media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 +2024-12-31 23:55:22,196 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:22,826 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:23,199 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:23,827 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:24,197 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:25,179 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:25,206 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:26,203 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:27,208 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:27,828 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:28,214 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:29,216 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:30,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:31,100 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:31,213 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:32,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:32,851 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:55:33,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:55:33,188 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:55:33,229 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:33,830 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:34,221 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:35,214 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:35,831 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:36,215 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:36,941 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:37,216 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:37,832 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:38,215 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:39,216 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:40,216 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:41,216 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:42,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:42,222 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:43,216 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:44,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:45,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:46,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:47,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:47,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:48,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:55:48,171 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:55:48,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:49,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:50,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:50,755 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:55:50,755 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:55:50,755 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:55:50,836 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:55:50,836 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:55:51,079 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/sq9owgqg-events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:55:51,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:51,836 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:52,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:52,756 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:53,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:54,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:55,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:56,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:57,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:57,838 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:58,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:58,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:59,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:00,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:01,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:01,840 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:02,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:02,851 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:56:03,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:56:03,171 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:56:03,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:03,287 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:03,840 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:04,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:05,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:06,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:07,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:08,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:09,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:09,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:10,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:11,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:12,221 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:13,221 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:14,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:14,221 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:15,221 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:16,222 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:17,222 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:18,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:56:18,171 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:56:18,222 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:19,222 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:19,298 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:20,222 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:21,224 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:21,847 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:22,213 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:56:22,213 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 with policy now +2024-12-31 23:56:22,215 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:56:22,216 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:56:22,216 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:56:22,216 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:56:22,235 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:22,500 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/wkbtqql3-media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 +2024-12-31 23:56:22,848 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:22,848 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:56:22,849 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 +2024-12-31 23:56:22,849 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval +2024-12-31 23:56:23,129 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/kbw2oye5-media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 +2024-12-31 23:56:23,231 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:23,848 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:24,238 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:24,849 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:25,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:25,239 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:26,236 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:27,238 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:28,240 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:29,250 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:30,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:30,245 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:31,246 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:32,253 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:32,852 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:56:33,179 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:56:33,179 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:56:33,244 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:33,851 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:34,248 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:35,255 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:35,305 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:35,852 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:36,246 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:37,253 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:37,853 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:38,250 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:39,255 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:40,250 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:41,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:41,253 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:41,854 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:56:42,249 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:42,596 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:56:42,597 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:56:42,597 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:56:42,854 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:56:43,253 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:43,855 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:44,254 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:45,253 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:46,258 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:46,780 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:47,257 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:47,856 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:48,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:56:48,177 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:56:48,257 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:49,257 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:49,857 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:50,257 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:51,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:51,857 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:52,104 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:52,258 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:53,258 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:53,858 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:54,258 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:55,258 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:56,258 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:57,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:57,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:58,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:59,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:00,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:01,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:02,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:02,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:02,852 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:57:03,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:57:03,173 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:57:03,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:04,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:05,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:06,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:07,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:07,321 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:08,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:09,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:10,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:11,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:12,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:13,182 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:13,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:14,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:15,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:16,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:17,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:18,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:57:18,173 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:57:18,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:18,263 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:19,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:20,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:21,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:22,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:23,262 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:23,524 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:24,263 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:25,263 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:25,869 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:26,265 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:26,929 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:57:26,929 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 with policy now +2024-12-31 23:57:26,929 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:57:26,930 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:57:26,930 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:57:26,932 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:57:27,207 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/8o71j225-media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 +2024-12-31 23:57:27,268 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:27,870 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:27,870 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:57:27,870 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 +2024-12-31 23:57:27,871 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval +2024-12-31 23:57:28,164 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/e6w1qhze-media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 +2024-12-31 23:57:28,270 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:28,870 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:29,197 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:29,273 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:30,280 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:31,287 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:32,282 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:32,853 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:57:33,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:57:33,190 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:57:33,281 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:33,872 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:34,280 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:34,345 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:35,282 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:36,280 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:37,286 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:38,283 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:39,286 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:39,874 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:40,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:40,285 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:41,282 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:41,875 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:42,282 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:43,288 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:44,283 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:45,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:45,285 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:46,287 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:46,672 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:57:46,673 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:57:46,673 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:57:46,876 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:57:46,877 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:57:47,145 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/e7fk64i2-events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:57:47,290 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:47,877 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:48,183 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:57:48,184 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:57:48,294 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:49,287 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:50,251 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:50,287 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:51,301 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:52,303 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:53,306 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:53,879 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:54,304 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:55,306 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:56,195 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:56,306 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:57,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:57,880 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:58,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:59,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:59,881 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:00,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:01,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:01,796 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:01,882 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:02,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:02,853 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:58:03,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:58:03,181 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:58:03,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:04,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:05,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:06,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:07,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:07,765 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:07,884 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:08,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:09,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:10,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:11,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:12,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:13,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:13,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:14,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:15,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:16,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:17,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:18,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:58:18,181 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:58:18,248 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:18,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:19,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:20,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:21,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:22,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:23,313 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:24,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:24,313 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:25,313 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:26,313 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:27,313 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:28,314 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:29,314 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:29,611 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:30,314 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:31,314 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:31,892 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:32,314 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:32,854 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:58:33,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:58:33,181 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:58:33,314 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:33,893 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:34,315 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:35,315 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:35,362 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:35,894 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:36,324 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:36,830 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:58:36,831 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 with policy now +2024-12-31 23:58:36,841 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:58:36,841 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:58:36,841 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:36,842 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:36,894 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:58:36,895 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 +2024-12-31 23:58:36,895 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval +2024-12-31 23:58:37,097 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/ibli9dry-media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 +2024-12-31 23:58:37,316 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:37,364 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/ijgpii4s-media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 +2024-12-31 23:58:37,895 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:38,323 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:38,895 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:39,326 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:40,325 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:41,205 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:41,321 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:42,331 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:43,335 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:43,897 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:44,341 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:45,339 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:46,208 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:46,337 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:47,335 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:47,898 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:48,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:58:48,187 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:58:48,338 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:49,344 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:49,899 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:50,345 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:51,348 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:51,900 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:51,977 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:52,353 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:53,364 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:54,347 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:54,751 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:58:54,751 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:54,752 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:54,901 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:58:54,901 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:58:55,170 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/mv7627av-events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:58:55,297 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:58:55,298 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:58:55,347 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:55,435 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:58:55,717 INFO wandb-upload_1:1770882 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpflqy7wn6 +2024-12-31 23:58:55,811 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:58:55,819 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:58:55,901 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:55,901 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:58:55,901 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json +2024-12-31 23:58:55,902 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2024-12-31 23:58:55,902 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media +2024-12-31 23:58:55,902 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval +2024-12-31 23:58:55,902 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table +2024-12-31 23:58:56,070 INFO SenderThread:1770882 [sender.py:send_request_log_artifact():1525] logged artifact run-71icjjya-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIxODQ5Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE4MDA0Ng==', 'latestArtifact': None}} +2024-12-31 23:58:56,071 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:58:56,071 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2024-12-31 23:58:56,071 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:58:56,071 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,072 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,072 DEBUG SenderThread:1770882 [sender.py:send():379] send: telemetry +2024-12-31 23:58:56,072 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,076 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,076 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,076 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,076 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,077 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,077 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,077 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,077 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,078 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,078 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,078 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,078 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,079 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,079 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,079 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,079 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,079 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,080 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,080 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,080 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,080 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,081 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:58:56,254 INFO wandb-upload_2:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/ci4ufzv5-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2024-12-31 23:58:56,295 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/hbnryvaw-media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json +2024-12-31 23:58:56,422 INFO wandb-upload_1:1770882 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0_8k8kau +2024-12-31 23:58:56,486 INFO wandb-upload_2:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/urwqbvs2-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2024-12-31 23:58:56,770 INFO SenderThread:1770882 [sender.py:send_request_log_artifact():1525] logged artifact run-71icjjya-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIxODQ5Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE4MDA0Nw==', 'latestArtifact': None}} +2024-12-31 23:58:56,770 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:58:56,770 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json with policy now +2024-12-31 23:58:56,770 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:58:56,770 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,771 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,771 DEBUG SenderThread:1770882 [sender.py:send():379] send: exit +2024-12-31 23:58:56,771 INFO SenderThread:1770882 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 23:58:56,771 INFO SenderThread:1770882 [sender.py:send_exit():588] handling runtime: 383 +2024-12-31 23:58:56,775 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,775 INFO SenderThread:1770882 [sender.py:send_exit():594] send defer +2024-12-31 23:58:56,775 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:58:56,775 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 23:58:56,776 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:58:56,776 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 23:58:56,776 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 1 +2024-12-31 23:58:56,776 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:58:56,776 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 23:58:56,776 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:58:56,776 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 23:58:56,776 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 2 +2024-12-31 23:58:56,776 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:58:56,776 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 23:58:56,776 INFO HandlerThread:1770882 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 23:58:56,776 DEBUG SystemMonitor:1770882 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 23:58:56,777 DEBUG SystemMonitor:1770882 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 23:58:56,777 INFO HandlerThread:1770882 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 23:58:56,777 INFO HandlerThread:1770882 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 23:58:56,830 INFO HandlerThread:1770882 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 23:58:56,831 INFO HandlerThread:1770882 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 23:58:56,831 INFO HandlerThread:1770882 [interfaces.py:finish():200] Joined network monitor +2024-12-31 23:58:56,831 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:58:56,831 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 23:58:56,831 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 3 +2024-12-31 23:58:56,831 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:58:56,832 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:58:56,832 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 23:58:56,832 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:58:56,832 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 23:58:56,832 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 4 +2024-12-31 23:58:56,832 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:58:56,832 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 23:58:56,902 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:58:57,009 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/11xrgokq-media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json +2024-12-31 23:58:57,902 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:59:03,755 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:59:03,755 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:59:03,755 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 23:59:03,755 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:59:03,756 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:03,756 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 23:59:03,756 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 5 +2024-12-31 23:59:03,756 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:59:03,756 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:59:03,756 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:59:03,757 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:03,759 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 23:59:03,760 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:59:03,760 DEBUG SenderThread:1770882 [sender.py:send():379] send: summary +2024-12-31 23:59:03,760 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:59:03,760 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:03,760 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 23:59:03,760 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 6 +2024-12-31 23:59:03,760 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:03,760 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 23:59:03,761 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:03,761 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 23:59:03,761 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 7 +2024-12-31 23:59:03,761 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:59:03,761 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:03,761 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 23:59:03,761 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:03,761 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 23:59:03,944 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:59:03,945 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/config.yaml +2024-12-31 23:59:04,238 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:59:06,771 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 8 +2024-12-31 23:59:06,771 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:59:06,772 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:06,772 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 23:59:06,772 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:06,772 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 23:59:06,772 INFO SenderThread:1770882 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 23:59:06,774 INFO SenderThread:1770882 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 23:59:06,776 INFO SenderThread:1770882 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 23:59:06,790 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 9 +2024-12-31 23:59:06,790 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:06,790 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 23:59:06,791 DEBUG SenderThread:1770882 [sender.py:send():379] send: artifact +2024-12-31 23:59:06,946 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:59:07,112 INFO wandb-upload_1:1770882 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp3bxld9c6 +2024-12-31 23:59:07,113 INFO wandb-upload_0:1770882 [upload_job.py:push():85] Skipped uploading /tmp/tmpw19jbu9c/wandb-job.json +2024-12-31 23:59:07,222 INFO wandb-upload_2:1770882 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpvmqc2ow2 +2024-12-31 23:59:07,238 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:59:07,872 INFO SenderThread:1770882 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIxODYyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDIwNzU2Mg==', 'versionIndex': 401}}} +2024-12-31 23:59:07,873 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:07,873 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 23:59:07,873 INFO SenderThread:1770882 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 23:59:07,947 INFO SenderThread:1770882 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files +2024-12-31 23:59:07,947 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/config.yaml config.yaml +2024-12-31 23:59:07,947 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json wandb-summary.json +2024-12-31 23:59:07,947 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/requirements.txt requirements.txt +2024-12-31 23:59:07,947 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-metadata.json wandb-metadata.json +2024-12-31 23:59:07,949 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:59:07,952 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log output.log +2024-12-31 23:59:07,953 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/diff.patch diff.patch +2024-12-31 23:59:07,953 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 23:59:07,956 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 10 +2024-12-31 23:59:07,957 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:59:07,957 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:07,957 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 23:59:07,958 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:07,958 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 23:59:07,958 INFO SenderThread:1770882 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:59:08,133 INFO wandb-upload_2:1770882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/requirements.txt +2024-12-31 23:59:08,143 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/config.yaml +2024-12-31 23:59:08,161 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:59:08,185 INFO wandb-upload_4:1770882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:59:08,239 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:59:08,239 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:59:08,253 INFO wandb-upload_3:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/cehsxfet-events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:59:08,453 INFO Thread-11 (_thread_body):1770882 [sender.py:transition_state():614] send defer: 11 +2024-12-31 23:59:08,453 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:08,453 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 23:59:08,454 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:08,454 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 23:59:08,454 INFO SenderThread:1770882 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 23:59:08,455 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 12 +2024-12-31 23:59:08,455 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:08,455 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 23:59:08,455 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:08,455 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 23:59:08,455 INFO SenderThread:1770882 [file_stream.py:finish():601] file stream finish called +2024-12-31 23:59:08,638 INFO SenderThread:1770882 [file_stream.py:finish():605] file stream finish is done +2024-12-31 23:59:08,638 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 13 +2024-12-31 23:59:08,638 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:08,638 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 23:59:08,639 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:08,639 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 23:59:08,639 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 14 +2024-12-31 23:59:08,639 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:08,639 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 23:59:08,639 DEBUG SenderThread:1770882 [sender.py:send():379] send: final +2024-12-31 23:59:08,639 DEBUG SenderThread:1770882 [sender.py:send():379] send: footer +2024-12-31 23:59:08,639 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:08,639 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 23:59:08,640 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:59:08,640 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:59:08,640 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 23:59:08,640 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:59:08,640 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 23:59:08,641 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 23:59:08,642 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: server_info +2024-12-31 23:59:08,643 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:59:08,694 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:59:08,695 INFO MainThread:1770882 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 23:59:08,695 INFO MainThread:1770882 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 23:59:08,695 INFO MainThread:1770882 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 23:59:08,695 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 23:59:08,696 INFO HandlerThread:1770882 [handler.py:finish():882] shutting down handler +2024-12-31 23:59:09,642 INFO WriterThread:1770882 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/run-71icjjya.wandb +2024-12-31 23:59:09,695 INFO SenderThread:1770882 [sender.py:finish():1615] shutting down sender +2024-12-31 23:59:09,695 INFO SenderThread:1770882 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:59:09,695 INFO SenderThread:1770882 [file_pusher.py:join():175] waiting for file pusher diff --git a/71icjjya/wandb/debug.log b/71icjjya/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..361c8c4ed5f49b8c914fe468c5a8ec6557d2f607 --- /dev/null +++ b/71icjjya/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-31 23:52:32,523 INFO MainThread:1770473 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 23:52:32,523 INFO MainThread:1770473 [wandb_setup.py:_flush():76] Configure stats pid to 1770473 +2024-12-31 23:52:32,523 INFO MainThread:1770473 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 23:52:32,523 INFO MainThread:1770473 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/settings +2024-12-31 23:52:32,523 INFO MainThread:1770473 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 23:52:32,523 INFO MainThread:1770473 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 23:52:32,523 INFO MainThread:1770473 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 23:52:32,523 INFO MainThread:1770473 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 23:52:32,523 INFO MainThread:1770473 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/logs/debug.log +2024-12-31 23:52:32,524 INFO MainThread:1770473 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/logs/debug-internal.log +2024-12-31 23:52:32,524 INFO MainThread:1770473 [wandb_init.py:init():569] calling init triggers +2024-12-31 23:52:32,524 INFO MainThread:1770473 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya'} +2024-12-31 23:52:32,524 INFO MainThread:1770473 [wandb_init.py:init():619] starting backend +2024-12-31 23:52:32,524 INFO MainThread:1770473 [wandb_init.py:init():623] setting up manager +2024-12-31 23:52:32,528 INFO MainThread:1770473 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 23:52:32,528 INFO MainThread:1770473 [wandb_init.py:init():631] backend started and connected +2024-12-31 23:52:32,544 INFO MainThread:1770473 [wandb_init.py:init():720] updated telemetry +2024-12-31 23:52:32,550 INFO MainThread:1770473 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 23:52:32,789 INFO MainThread:1770473 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 23:52:32,831 INFO MainThread:1770473 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 23:52:32,831 INFO MainThread:1770473 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 23:52:33,168 INFO MainThread:1770473 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 23:52:33,168 INFO MainThread:1770473 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 23:52:33,168 INFO MainThread:1770473 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 23:52:33,168 INFO MainThread:1770473 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 23:52:33,170 INFO MainThread:1770473 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 23:52:33,170 INFO MainThread:1770473 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/tb, True +2024-12-31 23:52:45,871 INFO MainThread:1770473 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2024-12-31 23:59:09,696 WARNING MsgRouterThr:1770473 [router.py:message_loop():77] message_loop has been closed diff --git a/71icjjya/wandb/run-20241231_235232-71icjjya/files/code/FISH/eval_robot.py b/71icjjya/wandb/run-20241231_235232-71icjjya/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/71icjjya/wandb/run-20241231_235232-71icjjya/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/71icjjya/wandb/run-20241231_235232-71icjjya/files/config.yaml b/71icjjya/wandb/run-20241231_235232-71icjjya/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c2123f996a1162b82e052344004061d59b0cd9d --- /dev/null +++ b/71icjjya/wandb/run-20241231_235232-71icjjya/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1745_0' +wandb_notes: + desc: null + value: 1745_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735707152 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/71icjjya/wandb/run-20241231_235232-71icjjya/files/diff.patch b/71icjjya/wandb/run-20241231_235232-71icjjya/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..7f9cbfaf74a18317e7403f01a665bc5e3d98239c --- /dev/null +++ b/71icjjya/wandb/run-20241231_235232-71icjjya/files/diff.patch @@ -0,0 +1,106 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..57a851f 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1007_0' ++# wandb_run_id: '1002_0' + ++# all books, 6/20 demos per book ++# RGBD+mask+act history ++# wandb_run_id: '1007_0' ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..50493fe 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -490,7 +490,8 @@ class Workspace: + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] +- total_num_episodes += filtered_run.summary_metrics['episode'] ++ # total_num_episodes += filtered_run.summary_metrics['episode'] ++ total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes +@@ -513,7 +514,6 @@ class Workspace: + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + +- + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index e3031af..6f97e26 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,11 +330,11 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.01 +- self.move_to_rot_tolerance = 2.0 ++ self.move_to_trans_tolerance = 0.005 ++ self.move_to_rot_tolerance = 1.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) +- self.move_to_target_timeout = 15.0 ++ self.move_to_target_timeout = 5.0 + self.move_to_target_max_rotation = 25.0 + # only can use one of these at a time + # assert use_roll + use_pitch + use_yaw <= 1, "Can only use one of roll, pitch, yaw" diff --git a/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json b/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json new file mode 100644 index 0000000000000000000000000000000000000000..30d70111fb6d2524d04966b9af5719ddb91b0b18 --- /dev/null +++ b/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.2, 0.003681928018651432, 0.7286170720596843, 0.0051474646723538384, 0.645829737157266, 0.025117173885908266, 0.5745371579974826, 0.10005546369206852, 0.4711491614637855]]} \ No newline at end of file diff --git a/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json b/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json new file mode 100644 index 0000000000000000000000000000000000000000..fc42bc5932f1c52295d548f9f2ca92ff57bb58e4 --- /dev/null +++ b/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[11, 30, 0.36666666666666664, 0.18215820199647548, 0.5829939246011686, 0.21580573922119123, 0.5600243657707329, 0.2377920076775446, 0.5129164904330169, 0.25260216375172906, 0.4966279752559255]]} \ No newline at end of file diff --git a/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 b/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ecdcd457c05e288107cfae93b7e71bd330e3a165 --- /dev/null +++ b/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65829ddfa6c31573c9e76629a013199ca03a049a3a08c05340a44cb6852c78be +size 1464590 diff --git a/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 b/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..31cbb80452406f3b5e6bdf02060ba1f97d2c92d7 --- /dev/null +++ b/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45cd68318424e8a1aa34a9f5ea973589e5135101a19cfe2f08f18f61f3851619 +size 1446227 diff --git a/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 b/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..860f9081fb6ea42456494d29b4bd6aa7431670c2 --- /dev/null +++ b/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0389095f97a9dda10a03d471d919762d282d87196599d44d2c89accb3edba1a7 +size 997527 diff --git a/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 b/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..471fda56802c5192782ea7e6893451eab913641f --- /dev/null +++ b/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe95f75420ab4bf1f2e0926922851a7e4b0701b9e9046a2025e526945b5389a9 +size 1488529 diff --git a/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 b/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0b7ff5e8f424f55c9ca2dcc100976e990c8a5197 --- /dev/null +++ b/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c467f37c076e27eccc382a3a0643d361211b905835828e0362c46589ed733e2 +size 1464399 diff --git a/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log b/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..754de5dc58d00c856aa2607b91cf237659cbc8a8 --- /dev/null +++ b/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log @@ -0,0 +1,318 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735707165.872190]: resetting environment +[INFO] [1735707165.876967]: cleared current plan +[INFO] [1735707165.878077]: moving to home +Move to target timeout: 5.0007030963897705. Trans error: 0.005429703276604414. Rot error: 0.5778504585686638 +Move to target timeout: 5.09975790977478. Trans error: 0.0053318578275194295. Rot error: 1.2864200155262016 +[INFO] [1735707176.981572]: reached home +[INFO] [1735707176.982756]: reset action history +[INFO] [1735707179.693869]: environment reset +Starting episode 0 +[INFO] [1735707179.698552]: resetting environment +[INFO] [1735707179.702810]: cleared current plan +[INFO] [1735707179.703608]: moving to home +Move to target timeout: 5.000932693481445. Trans error: 0.005365655986034889. Rot error: 1.289116478851478 +[INFO] [1735707185.707918]: reached home +[INFO] [1735707185.710652]: reset action history +[INFO] [1735707188.475839]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1735707200.751182]: resetting environment +[INFO] [1735707200.755447]: cleared current plan +[INFO] [1735707200.756335]: moving to home +Move to target timeout: 5.001586675643921. Trans error: 0.0053642822241811985. Rot error: 1.2877455227736294 +[INFO] [1735707206.760110]: reached home +[INFO] [1735707206.761271]: reset action history +[INFO] [1735707209.470167]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/episode_rosbags/episode_0_2024-12-31-23-53-29.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1770473.log +started roslaunch server http://158.130.50.37:39191/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1771036] +process[print_text-2]: started with pid [1771060] +process[rosbag_record-3]: started with pid [1771061] +started bagging! +For topic gripper_width: timestamp difference is 108612925 for nearest: 1735707241419973999 - target: 1735707241311361074 at idx 249 +[INFO] [1735707244.268898]: Storing episode... +[rosbag_record-3] killing on exit[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735707245.021819]: Stored episode 1. +[INFO] [1735707245.022184]: Saving video... +[INFO] [1735707245.217505]: Video saved! +Starting episode 1 +[INFO] [1735707245.232674]: resetting environment +[INFO] [1735707245.241817]: cleared current plan +[INFO] [1735707245.242179]: moving to home +Move to target timeout: 5.00079345703125. Trans error: 0.00783576164394617. Rot error: 1.234344250208114 +Move to target timeout: 5.09941029548645. Trans error: 0.005559350177709476. Rot error: 0.9493888529747027 +[INFO] [1735707256.345221]: reached home +[INFO] [1735707256.345893]: reset action history +[INFO] [1735707259.058944]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735707272.098331]: resetting environment +[INFO] [1735707272.106758]: cleared current plan +[INFO] [1735707272.106957]: moving to home +Move to target timeout: 5.000710487365723. Trans error: 0.005375167820602655. Rot error: 0.5447402232025141 +Move to target timeout: 5.099777936935425. Trans error: 0.005453417045954734. Rot error: 1.3384793012056364 +[INFO] [1735707283.209308]: reached home +[INFO] [1735707283.209934]: reset action history +[INFO] [1735707285.917755]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/episode_rosbags/episode_1_2024-12-31-23-54-45.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1770473.log +started roslaunch server http://158.130.50.37:39417/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1771213] +process[print_text-5]: started with pid [1771214] +process[rosbag_record-6]: started with pid [1771238] +started bagging! +For topic gripper_width: timestamp difference is -100701983 for nearest: 1735707312253271406 - target: 1735707312353973389 at idx 249 +For topic gripper_width: timestamp difference is 100359512 for nearest: 1735707313719981504 - target: 1735707313619621992 at idx 249 +For topic gripper_width: timestamp difference is 101325546 for nearest: 1735707315019944940 - target: 1735707314918619394 at idx 249 +preempting episode +[INFO] [1735707319.373213]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1735707320.166784]: Stored episode 2. +[INFO] [1735707320.167634]: Saving video... +[INFO] [1735707320.879273]: Video saved! +[WARN] [1735707320.547358]: Plan exhausted +[WARN] [1735707320.587600]: Plan exhausted +[WARN] [1735707320.627370]: Plan exhausted +[WARN] [1735707320.670496]: Plan exhausted +[WARN] [1735707320.709118]: Plan exhausted +[WARN] [1735707320.747943]: Plan exhausted +[WARN] [1735707320.787555]: Plan exhausted +[WARN] [1735707320.828533]: Plan exhausted +[WARN] [1735707320.867397]: Plan exhausted +[WARN] [1735707320.909407]: Plan exhausted +[WARN] [1735707320.970217]: Plan exhausted +[WARN] [1735707320.990975]: Plan exhausted +Starting episode 2 +[INFO] [1735707320.923710]: resetting environment +[INFO] [1735707320.994616]: cleared current plan +[INFO] [1735707320.997225]: moving to home +Move to target timeout: 5.00276517868042. Trans error: 0.005473780911415815. Rot error: 0.8229960847972231 +Move to target timeout: 5.098938941955566. Trans error: 0.005522250004281323. Rot error: 0.9453433184624774 +[INFO] [1735707332.109307]: reached home +[INFO] [1735707332.111621]: reset action history +[INFO] [1735707334.897657]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735707350.348881]: resetting environment +[INFO] [1735707350.354690]: cleared current plan +[INFO] [1735707350.355108]: moving to home +Move to target timeout: 5.001073837280273. Trans error: 0.005016516382213575. Rot error: 0.9821345622080134 +[INFO] [1735707356.357284]: reached home +[INFO] [1735707356.361792]: reset action history +[INFO] [1735707359.072350]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/episode_rosbags/episode_2_2024-12-31-23-55-59.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1770473.log +started roslaunch server http://158.130.50.37:43861/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1771388] +process[print_text-8]: started with pid [1771389] +process[rosbag_record-9]: started with pid [1771413] +started bagging! +For topic gripper_width: timestamp difference is 105249305 for nearest: 1735707361119997878 - target: 1735707361014748573 at idx 26 +preempting episode +[INFO] [1735707380.634679]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[WARN] [1735707381.934493]: Plan exhausted +[WARN] [1735707381.976057]: Plan exhausted +[WARN] [1735707382.020229]: Plan exhausted +[WARN] [1735707382.054130]: Plan exhausted +[WARN] [1735707382.093748]: Plan exhausted +[INFO] [1735707381.619094]: Stored episode 3. +[INFO] [1735707381.624002]: Saving video... +[INFO] [1735707382.179839]: Video saved! +Starting episode 3 +[INFO] [1735707382.221993]: resetting environment +[INFO] [1735707382.264002]: cleared current plan +[INFO] [1735707382.265473]: moving to home +[WARN] [1735707382.133806]: Plan exhausted +[WARN] [1735707382.173711]: Plan exhausted +[WARN] [1735707382.214546]: Plan exhausted +[WARN] [1735707382.253703]: Plan exhausted +Move to target timeout: 5.098124265670776. Trans error: 0.005467342205896778. Rot error: 0.8745293071522999 +[INFO] [1735707393.273133]: reached home +[INFO] [1735707393.277637]: reset action history +[INFO] [1735707396.055058]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735707401.770561]: resetting environment +[INFO] [1735707401.777457]: cleared current plan +[INFO] [1735707401.777659]: moving to home +Move to target timeout: 5.000852346420288. Trans error: 0.005120097975404439. Rot error: 1.0106479899512932 +[INFO] [1735707407.786739]: reached home +[INFO] [1735707407.790646]: reset action history +[INFO] [1735707410.560335]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/episode_rosbags/episode_3_2024-12-31-23-56-50.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1770473.log +started roslaunch server http://158.130.50.37:35559/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1771553] +process[print_text-11]: started with pid [1771554] +process[rosbag_record-12]: started with pid [1771555] +started bagging! +For topic gripper_width: timestamp difference is 105178653 for nearest: 1735707443519938289 - target: 1735707443414759636 at idx 249 +[INFO] [1735707445.561259]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1735707446.305790]: Stored episode 4. +[INFO] [1735707446.314599]: Saving video... +[INFO] [1735707446.854688]: Video saved! +Starting episode 4 +[INFO] [1735707446.943342]: resetting environment +[INFO] [1735707446.973710]: cleared current plan +[INFO] [1735707446.974859]: moving to home +[WARN] [1735707446.898861]: Plan exhausted +[WARN] [1735707446.934354]: Plan exhausted +Move to target timeout: 5.0007147789001465. Trans error: 0.008247057907283306. Rot error: 1.3043143544935116 +Move to target timeout: 5.099151611328125. Trans error: 0.005600116319527948. Rot error: 0.9384784013398816 +[INFO] [1735707458.090907]: reached home +[INFO] [1735707458.092961]: reset action history +[INFO] [1735707460.916466]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735707466.582210]: resetting environment +[INFO] [1735707466.605186]: cleared current plan +[INFO] [1735707466.611542]: moving to home +Move to target timeout: 5.0023033618927. Trans error: 0.005466385278850794. Rot error: 0.6009882513352409 +Move to target timeout: 5.083364725112915. Trans error: 0.005437994143987727. Rot error: 1.3119046530835474 +[INFO] [1735707477.731850]: reached home +[INFO] [1735707477.739496]: reset action history +[INFO] [1735707480.512568]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/episode_rosbags/episode_4_2024-12-31-23-58-00.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1770473.log +started roslaunch server http://158.130.50.37:33527/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1771676] +process[print_text-14]: started with pid [1771677] +process[rosbag_record-15]: started with pid [1771701] +started bagging! +For topic gripper_width: timestamp difference is 106762519 for nearest: 1735707486719938388 - target: 1735707486613175869 at idx 93 +For topic gripper_width: timestamp difference is -271543399 for nearest: 1735707509253269061 - target: 1735707509524812460 at idx 249 +For topic gripper_width: timestamp difference is -133947124 for nearest: 1735707511453273306 - target: 1735707511587220430 at idx 249 +For topic gripper_width: timestamp difference is -104279549 for nearest: 1735707514153271406 - target: 1735707514257550955 at idx 249 +[INFO] [1735707515.362422]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735707516.135404]: Stored episode 5. +[INFO] [1735707516.141165]: Saving video... +[INFO] [1735707516.744379]: Video saved! +[INFO] [1735707516.843134]: resetting environment +[INFO] [1735707516.857188]: cleared current plan +[INFO] [1735707516.858848]: moving to home +[WARN] [1735707516.693742]: Plan exhausted +[WARN] [1735707516.733814]: Plan exhausted +[WARN] [1735707516.776035]: Plan exhausted +[WARN] [1735707516.813906]: Plan exhausted +[WARN] [1735707516.853692]: Plan exhausted +Move to target timeout: 5.000849962234497. Trans error: 0.0054608494974672794. Rot error: 0.666686631115358 +Move to target timeout: 5.087454557418823. Trans error: 0.005559285954293113. Rot error: 0.8858348904462131 +[INFO] [1735707527.971894]: reached home +[INFO] [1735707527.979490]: reset action history +[INFO] [1735707530.927764]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/71icjjya/wandb/run-20241231_235232-71icjjya/files/requirements.txt b/71icjjya/wandb/run-20241231_235232-71icjjya/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/71icjjya/wandb/run-20241231_235232-71icjjya/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-metadata.json b/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..738e7252dd33939aa3505c169e51bd2113faacac --- /dev/null +++ b/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T04:52:32.858202", + "startedAt": "2025-01-01T04:52:32.521194", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.6215, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4497.485, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.702, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 621.5189628601074 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json b/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b28142a0c659f3e95455ec95887e9e849c21eafe --- /dev/null +++ b/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "65829ddfa6c31573c9e76629a013199ca03a049a3a08c05340a44cb6852c78be", "size": 1464590, "path": "media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4"}, "global_step": 608, "_timestamp": 1735707535.7189965, "_runtime": 383.1899514198303, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "45cd68318424e8a1aa34a9f5ea973589e5135101a19cfe2f08f18f61f3851619", "size": 1446227, "path": "media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4"}, "eval/num_success": 1.0, "episode": 30.0, "eval/success_rate": 0.20000000298023224, "eval/success_rate_lb_0.01": 0.0036819279193878174, "eval/success_rate_ub_0.01": 0.7286170721054077, "eval/success_rate_lb_0.025": 0.005147464573383331, "eval/success_rate_ub_0.025": 0.6458297371864319, "eval/success_rate_lb_0.05": 0.0251171737909317, "eval/success_rate_ub_0.05": 0.5745371580123901, "eval/success_rate_lb_0.1": 0.10005546361207962, "eval/success_rate_ub_0.1": 0.47114917635917664, "eval/2_eval": {"_type": "video-file", "sha256": "0389095f97a9dda10a03d471d919762d282d87196599d44d2c89accb3edba1a7", "size": 997527, "path": "media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "fe95f75420ab4bf1f2e0926922851a7e4b0701b9e9046a2025e526945b5389a9", "size": 1488529, "path": "media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "4c467f37c076e27eccc382a3a0643d361211b905835828e0362c46589ed733e2", "size": 1464399, "path": "media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "ff553b4df26f1e2b1b21e12412af8c8e47b3dc0662f8640d7c2b7298b63d3c98", "size": 405, "artifact_path": "wandb-client-artifact://58ka3f5erz9dnqpqzgp59gch1y02jwrlohp8exfzjic9gv1bu82f1csrzrwo7f16jxyslvzc6wtr4xa6534fb7dwg31xgl9aol31sbvge79bo8w8j6yy7et93nzmx45h/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://t4lxx0parbuchgtkaiap50q6bg71ngl9mbj8d7iuj94g9prb3tdn4vk8vjhlza18rid4z4cb2qd37v86f9qv59fa236snz9b5tm7e1h231dfoq038n977gsyqx56ob8c:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 11, "total_num_episodes": 30, "total_success_rate": 0.36666666666666664, "total_success_rate_lb_0.01": 0.18215820199647548, "total_success_rate_ub_0.01": 0.5829939246011686, "total_success_rate_lb_0.025": 0.21580573922119123, "total_success_rate_ub_0.025": 0.5600243657707329, "total_success_rate_lb_0.05": 0.2377920076775446, "total_success_rate_ub_0.05": 0.5129164904330169, "total_success_rate_lb_0.1": 0.25260216375172906, "total_success_rate_ub_0.1": 0.4966279752559255, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "f26da17f8eaa641b6744cecd3bf83e890353b37499c2f0b2055be85e22f0e462", "size": 524, "artifact_path": "wandb-client-artifact://7x6ebmcvnzmfvmdda3ziuskp4lh0oeg8wd5nawsm0u9mrz7r6xgvuyqg6s1k9ly9bl293p0eixcnwbm41utzgoubemnjm1zg19oew776arst37000tc177yipz1tx7i0/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://sxx89liw4ijkzfzgv46p7ckuio9ej21mzd156ilfxrfx1zntajfnkai2tun08ul4kpycmxaxjinm7lhswhl3h4fvaht7umzxtbys9guhfy3k426d62s606o78bcpudrl:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 383}, "eval/total_success_rate": 0.36666667461395264, "eval/total_success_rate_lb_0.01": 0.1821582019329071, "eval/total_success_rate_ub_0.01": 0.5829939246177673, "eval/total_success_rate_lb_0.025": 0.21580573916435242, "eval/total_success_rate_ub_0.025": 0.5600243806838989, "eval/total_success_rate_lb_0.05": 0.2377920150756836, "eval/total_success_rate_ub_0.05": 0.5129165053367615, "eval/total_success_rate_lb_0.1": 0.25260215997695923, "eval/total_success_rate_ub_0.1": 0.4966279864311218} \ No newline at end of file diff --git a/71icjjya/wandb/run-20241231_235232-71icjjya/logs/debug-internal.log b/71icjjya/wandb/run-20241231_235232-71icjjya/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6b1fe655b53ae0070063943c5dddf41f9cc428ac --- /dev/null +++ b/71icjjya/wandb/run-20241231_235232-71icjjya/logs/debug-internal.log @@ -0,0 +1,957 @@ +2024-12-31 23:52:32,529 INFO StreamThr :1770882 [internal.py:wandb_internal():85] W&B internal server running at pid: 1770882, started at: 2024-12-31 23:52:32.528874 +2024-12-31 23:52:32,531 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status +2024-12-31 23:52:32,545 INFO WriterThread:1770882 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/run-71icjjya.wandb +2024-12-31 23:52:32,545 DEBUG SenderThread:1770882 [sender.py:send():379] send: header +2024-12-31 23:52:32,552 DEBUG SenderThread:1770882 [sender.py:send():379] send: run +2024-12-31 23:52:32,774 INFO SenderThread:1770882 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files +2024-12-31 23:52:32,774 INFO SenderThread:1770882 [sender.py:_start_run_threads():1188] run started: 71icjjya with start time 1735707152.529045 +2024-12-31 23:52:32,789 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 23:52:32,790 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: check_version +2024-12-31 23:52:32,836 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 23:52:32,844 DEBUG HandlerThread:1770882 [system_info.py:__init__():26] System info init +2024-12-31 23:52:32,844 DEBUG HandlerThread:1770882 [system_info.py:__init__():41] System info init done +2024-12-31 23:52:32,845 INFO HandlerThread:1770882 [system_monitor.py:start():194] Starting system monitor +2024-12-31 23:52:32,845 INFO SystemMonitor:1770882 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 23:52:32,845 INFO HandlerThread:1770882 [system_monitor.py:probe():214] Collecting system info +2024-12-31 23:52:32,845 INFO SystemMonitor:1770882 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 23:52:32,846 INFO SystemMonitor:1770882 [interfaces.py:start():188] Started disk monitoring +2024-12-31 23:52:32,847 INFO SystemMonitor:1770882 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 23:52:32,847 INFO SystemMonitor:1770882 [interfaces.py:start():188] Started memory monitoring +2024-12-31 23:52:32,848 INFO SystemMonitor:1770882 [interfaces.py:start():188] Started network monitoring +2024-12-31 23:52:32,858 DEBUG HandlerThread:1770882 [system_info.py:probe():152] Probing system +2024-12-31 23:52:32,859 DEBUG HandlerThread:1770882 [system_info.py:_probe_git():137] Probing git +2024-12-31 23:52:32,864 DEBUG HandlerThread:1770882 [system_info.py:_probe_git():145] Probing git done +2024-12-31 23:52:32,864 DEBUG HandlerThread:1770882 [system_info.py:probe():200] Probing system done +2024-12-31 23:52:32,864 DEBUG HandlerThread:1770882 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T04:52:32.858202', 'startedAt': '2025-01-01T04:52:32.521194', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.6215, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.485, 'min': 800.0, 'max': 4700.0}, {'current': 4499.702, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 621.5189628601074}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 23:52:32,864 INFO HandlerThread:1770882 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 23:52:32,864 INFO HandlerThread:1770882 [system_monitor.py:probe():227] Publishing system info +2024-12-31 23:52:32,864 DEBUG HandlerThread:1770882 [system_info.py:_save_code():44] Saving code +2024-12-31 23:52:32,866 DEBUG HandlerThread:1770882 [system_info.py:_save_code():67] Saving code done +2024-12-31 23:52:32,866 DEBUG HandlerThread:1770882 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 23:52:32,927 DEBUG HandlerThread:1770882 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 23:52:32,928 INFO HandlerThread:1770882 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 23:52:32,931 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:52:32,931 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 23:52:32,932 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 23:52:32,932 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 23:52:33,166 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/5pwmmmvb-wandb-metadata.json +2024-12-31 23:52:33,167 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 23:52:33,167 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: python_packages +2024-12-31 23:52:33,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:52:33,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:33,169 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:52:33,171 INFO HandlerThread:1770882 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/tb" + save: true +} +control { + relay_id: "ed18d94dda074ba88639ea3a7fdd48f4" +} +_info { + stream_id: "71icjjya" +} + +2024-12-31 23:52:33,171 DEBUG HandlerThread:1770882 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 23:52:33,188 INFO wandb-upload_2:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/b3lguk4v-diff.patch +2024-12-31 23:52:33,223 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/vn55u65j-code/FISH/eval_robot.py +2024-12-31 23:52:33,229 DEBUG SenderThread:1770882 [sender.py:send():379] send: telemetry +2024-12-31 23:52:33,229 DEBUG SenderThread:1770882 [sender.py:send():379] send: tbrecord +2024-12-31 23:52:33,230 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:52:33,230 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 with policy live +2024-12-31 23:52:33,418 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/ioz6e7gv-events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:52:33,776 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:52:33,776 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:52:33,776 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-metadata.json +2024-12-31 23:52:33,777 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/requirements.txt +2024-12-31 23:52:33,777 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/code/FISH/eval_robot.py +2024-12-31 23:52:33,777 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/diff.patch +2024-12-31 23:52:33,777 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/code/FISH +2024-12-31 23:52:33,777 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/code +2024-12-31 23:52:34,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:35,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:35,775 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:52:36,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:37,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:38,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:38,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:39,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:40,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:41,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:42,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:43,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:43,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:44,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:45,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:45,872 DEBUG SenderThread:1770882 [sender.py:send():379] send: config +2024-12-31 23:52:46,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:47,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:47,779 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:52:48,167 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:52:48,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:52:48,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:48,235 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:49,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:50,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:51,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:52,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:53,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:53,780 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:52:54,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:54,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:55,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:56,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:57,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:57,781 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:52:58,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:59,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:59,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:59,782 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:00,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:01,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:01,783 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:02,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:03,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:53:03,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:53:03,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:04,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:04,224 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:04,784 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/config.yaml +2024-12-31 23:53:05,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:05,784 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:06,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:07,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:07,785 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:08,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:09,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:09,517 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:09,785 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:10,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:11,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:12,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:13,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:14,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:15,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:15,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:16,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:17,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:18,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:53:18,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:53:18,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:19,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:20,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:20,301 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:21,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:21,789 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:22,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:23,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:23,789 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:24,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:25,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:25,759 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:26,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:27,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:27,790 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:28,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:29,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:29,791 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:30,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:30,766 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:31,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:31,791 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:32,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:32,848 DEBUG SystemMonitor:1770882 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 23:53:32,849 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:53:33,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:53:33,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:53:33,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:34,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:35,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:36,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:36,541 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:37,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:38,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:39,176 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:40,176 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:41,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:42,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:42,176 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:43,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:44,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:45,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:46,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:47,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:48,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:53:48,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:53:48,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:48,269 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:49,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:50,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:51,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:52,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:53,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:54,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:54,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:55,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:56,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:57,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:58,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:59,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:59,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:00,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:01,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:02,179 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:02,849 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:54:03,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:54:03,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:54:03,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:03,799 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:04,179 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:04,269 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:05,185 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:05,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:54:05,262 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:54:05,262 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 with policy now +2024-12-31 23:54:05,262 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:54:05,262 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:54:05,263 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:54:05,535 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/abdxmvd5-media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 +2024-12-31 23:54:05,800 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:05,800 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 +2024-12-31 23:54:05,800 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:54:05,800 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos +2024-12-31 23:54:05,801 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval +2024-12-31 23:54:05,801 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media +2024-12-31 23:54:06,089 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/26nhcuhz-media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 +2024-12-31 23:54:06,184 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:07,183 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:07,801 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:08,183 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:09,184 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:10,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:10,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:11,185 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:11,802 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:12,185 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:13,185 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:14,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:15,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:15,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:16,185 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:17,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:17,804 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:18,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:54:18,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:54:18,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:19,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:20,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:21,101 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:21,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:21,805 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:22,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:23,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:24,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:25,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:26,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:26,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:27,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:28,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:29,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:30,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:31,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:31,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:32,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:32,810 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:54:32,850 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:54:33,049 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/s91prls9-events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:54:33,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:54:33,169 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:54:33,213 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:33,810 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:34,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:35,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:36,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:36,227 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:37,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:38,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:39,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:39,812 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:40,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:41,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:42,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:42,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:43,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:43,813 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:44,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:45,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:45,814 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:46,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:47,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:47,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:47,815 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:48,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:54:48,170 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:54:48,213 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:49,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:50,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:51,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:52,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:52,275 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:53,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:54,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:55,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:56,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:57,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:58,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:58,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:59,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:00,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:01,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:02,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:02,850 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:55:03,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:55:03,171 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:55:03,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:03,308 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:04,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:05,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:06,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:07,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:08,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:09,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:09,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:10,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:11,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:12,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:13,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:13,823 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:14,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:14,773 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:15,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:15,824 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:16,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:17,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:17,825 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:18,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:55:18,171 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:55:18,213 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:19,193 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:20,167 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:20,193 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:20,916 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:55:20,916 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 with policy now +2024-12-31 23:55:20,919 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:55:20,920 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:55:20,920 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:55:20,925 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:55:21,183 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/2citjgvy-media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 +2024-12-31 23:55:21,195 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:21,826 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:21,826 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:55:21,826 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 +2024-12-31 23:55:21,826 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval +2024-12-31 23:55:22,127 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/17gsyab6-media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 +2024-12-31 23:55:22,196 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:22,826 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:23,199 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:23,827 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:24,197 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:25,179 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:25,206 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:26,203 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:27,208 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:27,828 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:28,214 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:29,216 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:30,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:31,100 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:31,213 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:32,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:32,851 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:55:33,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:55:33,188 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:55:33,229 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:33,830 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:34,221 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:35,214 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:35,831 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:36,215 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:36,941 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:37,216 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:37,832 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:38,215 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:39,216 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:40,216 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:41,216 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:42,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:42,222 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:43,216 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:44,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:45,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:46,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:47,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:47,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:48,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:55:48,171 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:55:48,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:49,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:50,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:50,755 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:55:50,755 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:55:50,755 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:55:50,836 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:55:50,836 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:55:51,079 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/sq9owgqg-events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:55:51,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:51,836 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:52,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:52,756 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:53,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:54,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:55,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:56,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:57,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:57,838 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:58,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:58,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:59,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:00,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:01,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:01,840 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:02,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:02,851 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:56:03,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:56:03,171 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:56:03,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:03,287 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:03,840 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:04,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:05,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:06,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:07,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:08,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:09,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:09,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:10,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:11,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:12,221 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:13,221 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:14,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:14,221 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:15,221 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:16,222 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:17,222 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:18,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:56:18,171 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:56:18,222 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:19,222 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:19,298 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:20,222 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:21,224 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:21,847 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:22,213 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:56:22,213 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 with policy now +2024-12-31 23:56:22,215 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:56:22,216 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:56:22,216 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:56:22,216 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:56:22,235 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:22,500 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/wkbtqql3-media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 +2024-12-31 23:56:22,848 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:22,848 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:56:22,849 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 +2024-12-31 23:56:22,849 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval +2024-12-31 23:56:23,129 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/kbw2oye5-media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 +2024-12-31 23:56:23,231 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:23,848 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:24,238 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:24,849 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:25,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:25,239 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:26,236 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:27,238 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:28,240 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:29,250 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:30,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:30,245 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:31,246 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:32,253 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:32,852 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:56:33,179 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:56:33,179 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:56:33,244 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:33,851 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:34,248 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:35,255 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:35,305 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:35,852 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:36,246 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:37,253 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:37,853 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:38,250 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:39,255 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:40,250 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:41,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:41,253 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:41,854 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:56:42,249 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:42,596 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:56:42,597 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:56:42,597 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:56:42,854 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:56:43,253 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:43,855 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:44,254 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:45,253 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:46,258 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:46,780 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:47,257 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:47,856 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:48,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:56:48,177 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:56:48,257 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:49,257 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:49,857 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:50,257 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:51,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:51,857 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:52,104 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:52,258 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:53,258 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:53,858 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:54,258 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:55,258 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:56,258 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:57,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:57,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:58,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:59,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:00,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:01,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:02,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:02,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:02,852 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:57:03,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:57:03,173 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:57:03,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:04,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:05,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:06,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:07,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:07,321 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:08,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:09,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:10,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:11,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:12,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:13,182 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:13,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:14,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:15,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:16,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:17,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:18,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:57:18,173 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:57:18,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:18,263 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:19,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:20,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:21,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:22,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:23,262 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:23,524 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:24,263 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:25,263 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:25,869 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:26,265 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:26,929 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:57:26,929 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 with policy now +2024-12-31 23:57:26,929 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:57:26,930 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:57:26,930 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:57:26,932 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:57:27,207 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/8o71j225-media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 +2024-12-31 23:57:27,268 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:27,870 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:27,870 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:57:27,870 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 +2024-12-31 23:57:27,871 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval +2024-12-31 23:57:28,164 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/e6w1qhze-media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 +2024-12-31 23:57:28,270 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:28,870 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:29,197 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:29,273 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:30,280 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:31,287 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:32,282 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:32,853 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:57:33,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:57:33,190 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:57:33,281 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:33,872 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:34,280 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:34,345 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:35,282 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:36,280 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:37,286 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:38,283 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:39,286 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:39,874 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:40,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:40,285 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:41,282 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:41,875 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:42,282 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:43,288 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:44,283 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:45,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:45,285 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:46,287 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:46,672 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:57:46,673 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:57:46,673 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:57:46,876 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:57:46,877 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:57:47,145 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/e7fk64i2-events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:57:47,290 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:47,877 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:48,183 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:57:48,184 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:57:48,294 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:49,287 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:50,251 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:50,287 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:51,301 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:52,303 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:53,306 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:53,879 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:54,304 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:55,306 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:56,195 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:56,306 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:57,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:57,880 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:58,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:59,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:59,881 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:00,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:01,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:01,796 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:01,882 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:02,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:02,853 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:58:03,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:58:03,181 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:58:03,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:04,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:05,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:06,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:07,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:07,765 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:07,884 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:08,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:09,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:10,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:11,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:12,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:13,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:13,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:14,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:15,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:16,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:17,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:18,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:58:18,181 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:58:18,248 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:18,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:19,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:20,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:21,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:22,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:23,313 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:24,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:24,313 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:25,313 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:26,313 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:27,313 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:28,314 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:29,314 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:29,611 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:30,314 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:31,314 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:31,892 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:32,314 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:32,854 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:58:33,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:58:33,181 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:58:33,314 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:33,893 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:34,315 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:35,315 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:35,362 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:35,894 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:36,324 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:36,830 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:58:36,831 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 with policy now +2024-12-31 23:58:36,841 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:58:36,841 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:58:36,841 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:36,842 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:36,894 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:58:36,895 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 +2024-12-31 23:58:36,895 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval +2024-12-31 23:58:37,097 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/ibli9dry-media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 +2024-12-31 23:58:37,316 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:37,364 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/ijgpii4s-media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 +2024-12-31 23:58:37,895 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:38,323 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:38,895 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:39,326 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:40,325 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:41,205 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:41,321 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:42,331 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:43,335 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:43,897 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:44,341 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:45,339 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:46,208 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:46,337 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:47,335 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:47,898 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:48,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:58:48,187 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:58:48,338 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:49,344 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:49,899 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:50,345 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:51,348 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:51,900 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:51,977 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:52,353 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:53,364 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:54,347 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:54,751 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:58:54,751 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:54,752 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:54,901 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:58:54,901 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:58:55,170 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/mv7627av-events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:58:55,297 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:58:55,298 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:58:55,347 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:55,435 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:58:55,717 INFO wandb-upload_1:1770882 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpflqy7wn6 +2024-12-31 23:58:55,811 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:58:55,819 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:58:55,901 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:55,901 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:58:55,901 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json +2024-12-31 23:58:55,902 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2024-12-31 23:58:55,902 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media +2024-12-31 23:58:55,902 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval +2024-12-31 23:58:55,902 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table +2024-12-31 23:58:56,070 INFO SenderThread:1770882 [sender.py:send_request_log_artifact():1525] logged artifact run-71icjjya-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIxODQ5Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE4MDA0Ng==', 'latestArtifact': None}} +2024-12-31 23:58:56,071 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:58:56,071 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2024-12-31 23:58:56,071 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:58:56,071 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,072 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,072 DEBUG SenderThread:1770882 [sender.py:send():379] send: telemetry +2024-12-31 23:58:56,072 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,076 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,076 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,076 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,076 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,077 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,077 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,077 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,077 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,078 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,078 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,078 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,078 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,079 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,079 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,079 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,079 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,079 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,080 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,080 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,080 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,080 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,081 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:58:56,254 INFO wandb-upload_2:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/ci4ufzv5-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2024-12-31 23:58:56,295 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/hbnryvaw-media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json +2024-12-31 23:58:56,422 INFO wandb-upload_1:1770882 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0_8k8kau +2024-12-31 23:58:56,486 INFO wandb-upload_2:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/urwqbvs2-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2024-12-31 23:58:56,770 INFO SenderThread:1770882 [sender.py:send_request_log_artifact():1525] logged artifact run-71icjjya-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIxODQ5Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE4MDA0Nw==', 'latestArtifact': None}} +2024-12-31 23:58:56,770 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:58:56,770 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json with policy now +2024-12-31 23:58:56,770 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:58:56,770 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,771 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,771 DEBUG SenderThread:1770882 [sender.py:send():379] send: exit +2024-12-31 23:58:56,771 INFO SenderThread:1770882 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 23:58:56,771 INFO SenderThread:1770882 [sender.py:send_exit():588] handling runtime: 383 +2024-12-31 23:58:56,775 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,775 INFO SenderThread:1770882 [sender.py:send_exit():594] send defer +2024-12-31 23:58:56,775 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:58:56,775 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 23:58:56,776 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:58:56,776 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 23:58:56,776 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 1 +2024-12-31 23:58:56,776 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:58:56,776 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 23:58:56,776 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:58:56,776 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 23:58:56,776 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 2 +2024-12-31 23:58:56,776 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:58:56,776 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 23:58:56,776 INFO HandlerThread:1770882 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 23:58:56,776 DEBUG SystemMonitor:1770882 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 23:58:56,777 DEBUG SystemMonitor:1770882 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 23:58:56,777 INFO HandlerThread:1770882 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 23:58:56,777 INFO HandlerThread:1770882 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 23:58:56,830 INFO HandlerThread:1770882 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 23:58:56,831 INFO HandlerThread:1770882 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 23:58:56,831 INFO HandlerThread:1770882 [interfaces.py:finish():200] Joined network monitor +2024-12-31 23:58:56,831 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:58:56,831 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 23:58:56,831 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 3 +2024-12-31 23:58:56,831 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:58:56,832 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:58:56,832 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 23:58:56,832 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:58:56,832 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 23:58:56,832 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 4 +2024-12-31 23:58:56,832 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:58:56,832 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 23:58:56,902 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:58:57,009 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/11xrgokq-media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json +2024-12-31 23:58:57,902 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:59:03,755 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:59:03,755 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:59:03,755 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 23:59:03,755 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:59:03,756 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:03,756 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 23:59:03,756 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 5 +2024-12-31 23:59:03,756 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:59:03,756 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:59:03,756 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:59:03,757 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:03,759 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 23:59:03,760 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:59:03,760 DEBUG SenderThread:1770882 [sender.py:send():379] send: summary +2024-12-31 23:59:03,760 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:59:03,760 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:03,760 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 23:59:03,760 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 6 +2024-12-31 23:59:03,760 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:03,760 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 23:59:03,761 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:03,761 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 23:59:03,761 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 7 +2024-12-31 23:59:03,761 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:59:03,761 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:03,761 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 23:59:03,761 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:03,761 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 23:59:03,944 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:59:03,945 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/config.yaml +2024-12-31 23:59:04,238 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:59:06,771 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 8 +2024-12-31 23:59:06,771 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:59:06,772 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:06,772 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 23:59:06,772 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:06,772 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 23:59:06,772 INFO SenderThread:1770882 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 23:59:06,774 INFO SenderThread:1770882 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 23:59:06,776 INFO SenderThread:1770882 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 23:59:06,790 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 9 +2024-12-31 23:59:06,790 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:06,790 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 23:59:06,791 DEBUG SenderThread:1770882 [sender.py:send():379] send: artifact +2024-12-31 23:59:06,946 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:59:07,112 INFO wandb-upload_1:1770882 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp3bxld9c6 +2024-12-31 23:59:07,113 INFO wandb-upload_0:1770882 [upload_job.py:push():85] Skipped uploading /tmp/tmpw19jbu9c/wandb-job.json +2024-12-31 23:59:07,222 INFO wandb-upload_2:1770882 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpvmqc2ow2 +2024-12-31 23:59:07,238 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:59:07,872 INFO SenderThread:1770882 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIxODYyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDIwNzU2Mg==', 'versionIndex': 401}}} +2024-12-31 23:59:07,873 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:07,873 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 23:59:07,873 INFO SenderThread:1770882 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 23:59:07,947 INFO SenderThread:1770882 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files +2024-12-31 23:59:07,947 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/config.yaml config.yaml +2024-12-31 23:59:07,947 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json wandb-summary.json +2024-12-31 23:59:07,947 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/requirements.txt requirements.txt +2024-12-31 23:59:07,947 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-metadata.json wandb-metadata.json +2024-12-31 23:59:07,949 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:59:07,952 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log output.log +2024-12-31 23:59:07,953 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/diff.patch diff.patch +2024-12-31 23:59:07,953 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 23:59:07,956 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 10 +2024-12-31 23:59:07,957 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:59:07,957 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:07,957 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 23:59:07,958 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:07,958 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 23:59:07,958 INFO SenderThread:1770882 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:59:08,133 INFO wandb-upload_2:1770882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/requirements.txt +2024-12-31 23:59:08,143 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/config.yaml +2024-12-31 23:59:08,161 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:59:08,185 INFO wandb-upload_4:1770882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:59:08,239 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:59:08,239 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:59:08,253 INFO wandb-upload_3:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/cehsxfet-events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:59:08,453 INFO Thread-11 (_thread_body):1770882 [sender.py:transition_state():614] send defer: 11 +2024-12-31 23:59:08,453 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:08,453 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 23:59:08,454 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:08,454 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 23:59:08,454 INFO SenderThread:1770882 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 23:59:08,455 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 12 +2024-12-31 23:59:08,455 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:08,455 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 23:59:08,455 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:08,455 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 23:59:08,455 INFO SenderThread:1770882 [file_stream.py:finish():601] file stream finish called +2024-12-31 23:59:08,638 INFO SenderThread:1770882 [file_stream.py:finish():605] file stream finish is done +2024-12-31 23:59:08,638 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 13 +2024-12-31 23:59:08,638 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:08,638 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 23:59:08,639 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:08,639 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 23:59:08,639 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 14 +2024-12-31 23:59:08,639 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:08,639 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 23:59:08,639 DEBUG SenderThread:1770882 [sender.py:send():379] send: final +2024-12-31 23:59:08,639 DEBUG SenderThread:1770882 [sender.py:send():379] send: footer +2024-12-31 23:59:08,639 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:08,639 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 23:59:08,640 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:59:08,640 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:59:08,640 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 23:59:08,640 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:59:08,640 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 23:59:08,641 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 23:59:08,642 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: server_info +2024-12-31 23:59:08,643 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:59:08,694 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:59:08,695 INFO MainThread:1770882 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 23:59:08,695 INFO MainThread:1770882 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 23:59:08,695 INFO MainThread:1770882 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 23:59:08,695 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 23:59:08,696 INFO HandlerThread:1770882 [handler.py:finish():882] shutting down handler +2024-12-31 23:59:09,642 INFO WriterThread:1770882 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/run-71icjjya.wandb +2024-12-31 23:59:09,695 INFO SenderThread:1770882 [sender.py:finish():1615] shutting down sender +2024-12-31 23:59:09,695 INFO SenderThread:1770882 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:59:09,695 INFO SenderThread:1770882 [file_pusher.py:join():175] waiting for file pusher diff --git a/71icjjya/wandb/run-20241231_235232-71icjjya/logs/debug.log b/71icjjya/wandb/run-20241231_235232-71icjjya/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..361c8c4ed5f49b8c914fe468c5a8ec6557d2f607 --- /dev/null +++ b/71icjjya/wandb/run-20241231_235232-71icjjya/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-31 23:52:32,523 INFO MainThread:1770473 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 23:52:32,523 INFO MainThread:1770473 [wandb_setup.py:_flush():76] Configure stats pid to 1770473 +2024-12-31 23:52:32,523 INFO MainThread:1770473 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 23:52:32,523 INFO MainThread:1770473 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/settings +2024-12-31 23:52:32,523 INFO MainThread:1770473 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 23:52:32,523 INFO MainThread:1770473 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 23:52:32,523 INFO MainThread:1770473 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 23:52:32,523 INFO MainThread:1770473 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 23:52:32,523 INFO MainThread:1770473 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/logs/debug.log +2024-12-31 23:52:32,524 INFO MainThread:1770473 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/logs/debug-internal.log +2024-12-31 23:52:32,524 INFO MainThread:1770473 [wandb_init.py:init():569] calling init triggers +2024-12-31 23:52:32,524 INFO MainThread:1770473 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya'} +2024-12-31 23:52:32,524 INFO MainThread:1770473 [wandb_init.py:init():619] starting backend +2024-12-31 23:52:32,524 INFO MainThread:1770473 [wandb_init.py:init():623] setting up manager +2024-12-31 23:52:32,528 INFO MainThread:1770473 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 23:52:32,528 INFO MainThread:1770473 [wandb_init.py:init():631] backend started and connected +2024-12-31 23:52:32,544 INFO MainThread:1770473 [wandb_init.py:init():720] updated telemetry +2024-12-31 23:52:32,550 INFO MainThread:1770473 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 23:52:32,789 INFO MainThread:1770473 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 23:52:32,831 INFO MainThread:1770473 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 23:52:32,831 INFO MainThread:1770473 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 23:52:33,168 INFO MainThread:1770473 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 23:52:33,168 INFO MainThread:1770473 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 23:52:33,168 INFO MainThread:1770473 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 23:52:33,168 INFO MainThread:1770473 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 23:52:33,170 INFO MainThread:1770473 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 23:52:33,170 INFO MainThread:1770473 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/tb, True +2024-12-31 23:52:45,871 INFO MainThread:1770473 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2024-12-31 23:59:09,696 WARNING MsgRouterThr:1770473 [router.py:message_loop():77] message_loop has been closed diff --git a/71icjjya/wandb/run-20241231_235232-71icjjya/run-71icjjya.wandb b/71icjjya/wandb/run-20241231_235232-71icjjya/run-71icjjya.wandb new file mode 100644 index 0000000000000000000000000000000000000000..fbac74a425d4afaf13f9b00971a74f115d8859cb Binary files /dev/null and b/71icjjya/wandb/run-20241231_235232-71icjjya/run-71icjjya.wandb differ diff --git a/bufkm1dk/.hydra/config.yaml b/bufkm1dk/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57f0faa93e8c5310ed098ffd123244267c23fcf8 --- /dev/null +++ b/bufkm1dk/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1745_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/bufkm1dk/.hydra/hydra.yaml b/bufkm1dk/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0e08b76e29fa83ab044c0ca3d4b70f6c2ce412b --- /dev/null +++ b/bufkm1dk/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/bufkm1dk/.hydra/overrides.yaml b/bufkm1dk/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/bufkm1dk/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/bufkm1dk/episode_rosbags/aligned_depth_to_color_K.npy b/bufkm1dk/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/bufkm1dk/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/bufkm1dk/episode_rosbags/cam_tf_world.npy b/bufkm1dk/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/bufkm1dk/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/bufkm1dk/episode_rosbags/color_K.npy b/bufkm1dk/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/bufkm1dk/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/bufkm1dk/episode_rosbags/depth_K.npy b/bufkm1dk/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/bufkm1dk/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/bufkm1dk/episode_rosbags/episode_0_2024-12-31-21-55-35.bag b/bufkm1dk/episode_rosbags/episode_0_2024-12-31-21-55-35.bag new file mode 100644 index 0000000000000000000000000000000000000000..1fdb2ff9a9bec7c686e4ba4dc71cf170bdff354a --- /dev/null +++ b/bufkm1dk/episode_rosbags/episode_0_2024-12-31-21-55-35.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce283298025bf073c8befd3f3946304c47e6e70f0f7a845641af4594a20f7c55 +size 1064895315 diff --git a/bufkm1dk/episode_rosbags/episode_1_2024-12-31-21-56-11.bag b/bufkm1dk/episode_rosbags/episode_1_2024-12-31-21-56-11.bag new file mode 100644 index 0000000000000000000000000000000000000000..c40ab2776d055cb3356ae2829b80248b72976159 --- /dev/null +++ b/bufkm1dk/episode_rosbags/episode_1_2024-12-31-21-56-11.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57db5c97a984dcb8686a5e024260e863c94e091bfd35a92c6a55fb71fc79a831 +size 947812269 diff --git a/bufkm1dk/episode_rosbags/episode_2_2024-12-31-21-56-46.bag b/bufkm1dk/episode_rosbags/episode_2_2024-12-31-21-56-46.bag new file mode 100644 index 0000000000000000000000000000000000000000..729c76373b0bb9e718685d70bf7d0d359e993c38 --- /dev/null +++ b/bufkm1dk/episode_rosbags/episode_2_2024-12-31-21-56-46.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:850402f0f913b50857cf65c1b65a8886185ed600eb9d11eac74d1a0bd125af2e +size 985567486 diff --git a/bufkm1dk/episode_rosbags/episode_3_2024-12-31-21-57-18.bag b/bufkm1dk/episode_rosbags/episode_3_2024-12-31-21-57-18.bag new file mode 100644 index 0000000000000000000000000000000000000000..2afca3678b4e4a8d1a0916789c42db1e138fe4d3 --- /dev/null +++ b/bufkm1dk/episode_rosbags/episode_3_2024-12-31-21-57-18.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe182baf904aaf3ab5b8aac84869fbd848662a93c40195db55235dbec8b97ba2 +size 953195262 diff --git a/bufkm1dk/episode_rosbags/episode_4_2024-12-31-21-57-51.bag b/bufkm1dk/episode_rosbags/episode_4_2024-12-31-21-57-51.bag new file mode 100644 index 0000000000000000000000000000000000000000..65db7e6c42eb53cb1fbdd011fe74d1fb558d4afa --- /dev/null +++ b/bufkm1dk/episode_rosbags/episode_4_2024-12-31-21-57-51.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30caaf60b6d1ef225692857e0d0e05cf153497476414fd3bcf15574aebd36cb6 +size 997455560 diff --git a/bufkm1dk/eval_robot.log b/bufkm1dk/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..16a9580dd6bea688ee16ca0e6e68f9e22d900c76 --- /dev/null +++ b/bufkm1dk/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-31 21:55:06,804][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-31 21:55:06,807][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/bufkm1dk/eval_video/0_eval.mp4 b/bufkm1dk/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7f9f9dc563ef3eeafe6cedf852d72b76a5434eb9 --- /dev/null +++ b/bufkm1dk/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4df030dc74284b36be4e4a4c34dde9440321e9ec9fb2195051a8468e1c3f90c8 +size 659178 diff --git a/bufkm1dk/eval_video/1_eval.mp4 b/bufkm1dk/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..af119e619c8883a98d12ed7ae9a4fb70215aa80d --- /dev/null +++ b/bufkm1dk/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d43c0fcec8d2ec15ec24dc9e91446c61e517a8ca1dc162cd788f815862f3d2f +size 592382 diff --git a/bufkm1dk/eval_video/2_eval.mp4 b/bufkm1dk/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c7580cd492561b038b1cc8a638c696244eb18582 --- /dev/null +++ b/bufkm1dk/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9be307d2b8f293d1a04293215598d5c3e8749d52c6526647f95f8bb87b4b746 +size 610659 diff --git a/bufkm1dk/eval_video/3_eval.mp4 b/bufkm1dk/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..40c08a1ca28828c524b3d6439e3643f5827a55e8 --- /dev/null +++ b/bufkm1dk/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db8eec7ff63d941b6e62d54aec062994dfd9909882e1b34f5b904dcdf29a14db +size 601924 diff --git a/bufkm1dk/eval_video/4_eval.mp4 b/bufkm1dk/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5893049559bcd809857b99ab25da1b90474c7b5e --- /dev/null +++ b/bufkm1dk/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2da638a425088a67de89709460e61194fdc7ea1bfb52bf3e1bcc12dd05f07021 +size 620944 diff --git a/bufkm1dk/tb/events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 b/bufkm1dk/tb/events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 new file mode 100644 index 0000000000000000000000000000000000000000..d6a21c4a335c6e23c3ff76f5b32a75738154420e --- /dev/null +++ b/bufkm1dk/tb/events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e212d510f1bc1f42f0f251080414fa03c0e0f00ddcc114c9e52093d66ac34fd7 +size 6499 diff --git a/bufkm1dk/wandb/debug-internal.log b/bufkm1dk/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..95cb8fead48f5a61b6f023cd3d48231b20849487 --- /dev/null +++ b/bufkm1dk/wandb/debug-internal.log @@ -0,0 +1,671 @@ +2024-12-31 21:55:12,775 INFO StreamThr :1752284 [internal.py:wandb_internal():85] W&B internal server running at pid: 1752284, started at: 2024-12-31 21:55:12.774562 +2024-12-31 21:55:12,775 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status +2024-12-31 21:55:12,789 INFO WriterThread:1752284 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/run-bufkm1dk.wandb +2024-12-31 21:55:12,790 DEBUG SenderThread:1752284 [sender.py:send():379] send: header +2024-12-31 21:55:12,798 DEBUG SenderThread:1752284 [sender.py:send():379] send: run +2024-12-31 21:55:13,200 INFO SenderThread:1752284 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files +2024-12-31 21:55:13,200 INFO SenderThread:1752284 [sender.py:_start_run_threads():1188] run started: bufkm1dk with start time 1735700112.774932 +2024-12-31 21:55:13,217 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 21:55:13,218 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: check_version +2024-12-31 21:55:13,293 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 21:55:13,298 DEBUG HandlerThread:1752284 [system_info.py:__init__():26] System info init +2024-12-31 21:55:13,298 DEBUG HandlerThread:1752284 [system_info.py:__init__():41] System info init done +2024-12-31 21:55:13,298 INFO HandlerThread:1752284 [system_monitor.py:start():194] Starting system monitor +2024-12-31 21:55:13,298 INFO SystemMonitor:1752284 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 21:55:13,298 INFO HandlerThread:1752284 [system_monitor.py:probe():214] Collecting system info +2024-12-31 21:55:13,299 INFO SystemMonitor:1752284 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 21:55:13,299 INFO SystemMonitor:1752284 [interfaces.py:start():188] Started disk monitoring +2024-12-31 21:55:13,300 INFO SystemMonitor:1752284 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 21:55:13,301 INFO SystemMonitor:1752284 [interfaces.py:start():188] Started memory monitoring +2024-12-31 21:55:13,301 INFO SystemMonitor:1752284 [interfaces.py:start():188] Started network monitoring +2024-12-31 21:55:13,311 DEBUG HandlerThread:1752284 [system_info.py:probe():152] Probing system +2024-12-31 21:55:13,312 DEBUG HandlerThread:1752284 [system_info.py:_probe_git():137] Probing git +2024-12-31 21:55:13,318 DEBUG HandlerThread:1752284 [system_info.py:_probe_git():145] Probing git done +2024-12-31 21:55:13,318 DEBUG HandlerThread:1752284 [system_info.py:probe():200] Probing system done +2024-12-31 21:55:13,318 DEBUG HandlerThread:1752284 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T02:55:13.311205', 'startedAt': '2025-01-01T02:55:12.762423', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.3635000000004, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4492.45, 'min': 800.0, 'max': 4700.0}, {'current': 4500.003, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 530.4097137451172}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 21:55:13,319 INFO HandlerThread:1752284 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 21:55:13,319 INFO HandlerThread:1752284 [system_monitor.py:probe():227] Publishing system info +2024-12-31 21:55:13,319 DEBUG HandlerThread:1752284 [system_info.py:_save_code():44] Saving code +2024-12-31 21:55:13,320 DEBUG HandlerThread:1752284 [system_info.py:_save_code():67] Saving code done +2024-12-31 21:55:13,320 DEBUG HandlerThread:1752284 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 21:55:13,384 DEBUG HandlerThread:1752284 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 21:55:13,384 INFO HandlerThread:1752284 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 21:55:13,387 DEBUG SenderThread:1752284 [sender.py:send():379] send: files +2024-12-31 21:55:13,388 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 21:55:13,388 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 21:55:13,388 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 21:55:13,635 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 21:55:13,635 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: python_packages +2024-12-31 21:55:13,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:55:13,636 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:55:13,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:13,638 INFO HandlerThread:1752284 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/tb" + save: true +} +control { + relay_id: "10baa15bb7d642b7af75b0e9ed3f9ec7" +} +_info { + stream_id: "bufkm1dk" +} + +2024-12-31 21:55:13,638 DEBUG HandlerThread:1752284 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 21:55:13,680 INFO wandb-upload_2:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/764pzyax-diff.patch +2024-12-31 21:55:13,680 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/8qfd83d2-wandb-metadata.json +2024-12-31 21:55:13,744 DEBUG SenderThread:1752284 [sender.py:send():379] send: telemetry +2024-12-31 21:55:13,744 DEBUG SenderThread:1752284 [sender.py:send():379] send: tbrecord +2024-12-31 21:55:13,745 DEBUG SenderThread:1752284 [sender.py:send():379] send: files +2024-12-31 21:55:13,745 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 with policy live +2024-12-31 21:55:13,936 INFO wandb-upload_2:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/rur3i32c-events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:55:13,994 INFO wandb-upload_1:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/lp8puosj-code/FISH/eval_robot.py +2024-12-31 21:55:14,201 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/diff.patch +2024-12-31 21:55:14,201 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:55:14,202 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/code/FISH/eval_robot.py +2024-12-31 21:55:14,202 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-metadata.json +2024-12-31 21:55:14,202 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:14,202 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/requirements.txt +2024-12-31 21:55:14,202 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/code +2024-12-31 21:55:14,202 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/code/FISH +2024-12-31 21:55:14,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:15,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:16,200 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:16,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:17,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:18,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:55:18,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:19,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:20,039 DEBUG SenderThread:1752284 [sender.py:send():379] send: config +2024-12-31 21:55:20,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:21,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:22,202 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:22,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:23,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:55:23,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:24,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:25,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:26,203 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:26,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:27,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:28,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:55:28,636 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:55:28,677 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:28,733 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:55:29,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:30,204 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:30,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:31,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:32,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:33,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:34,206 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:34,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:55:34,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:35,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:36,206 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:36,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:37,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:38,207 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:38,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:39,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:55:39,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:40,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:41,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:42,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:43,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:55:43,636 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:55:43,677 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:44,640 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:44,725 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:55:45,209 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/config.yaml +2024-12-31 21:55:45,640 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:46,640 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:47,640 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:48,640 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:49,641 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:50,503 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:55:50,641 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:51,497 DEBUG SenderThread:1752284 [sender.py:send():379] send: files +2024-12-31 21:55:51,497 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_4df030dc74284b36be4e.mp4 with policy now +2024-12-31 21:55:51,499 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:55:51,500 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:55:51,500 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:55:51,501 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:55:51,641 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:51,743 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/awawrot7-media/videos/eval/0_eval_0_4df030dc74284b36be4e.mp4 +2024-12-31 21:55:52,211 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:52,211 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/0_eval_0_4df030dc74284b36be4e.mp4 +2024-12-31 21:55:52,211 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:55:52,211 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media +2024-12-31 21:55:52,211 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos +2024-12-31 21:55:52,211 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval +2024-12-31 21:55:52,489 INFO wandb-upload_1:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/qg8ywmar-media/videos/eval/0_eval_0_4df030dc74284b36be4e.mp4 +2024-12-31 21:55:52,641 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:53,641 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:54,212 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:54,641 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:55,641 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:55,857 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:55:56,213 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:56,641 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:57,641 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:58,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:55:58,636 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:55:58,677 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:59,643 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:00,214 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:00,643 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:01,643 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:01,739 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:02,643 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:03,644 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:04,644 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:05,644 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:06,644 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:07,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:07,644 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:08,217 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:56:08,644 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:09,644 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:10,218 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:10,645 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:11,645 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:12,645 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:13,076 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:13,301 DEBUG SystemMonitor:1752284 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 21:56:13,302 DEBUG SenderThread:1752284 [sender.py:send():379] send: stats +2024-12-31 21:56:13,665 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:56:13,665 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:13,665 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:56:14,219 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:14,645 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:15,645 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:16,219 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:16,645 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:17,646 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:18,220 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:18,269 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:18,646 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:19,646 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:20,646 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:21,646 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:22,647 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:23,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:23,647 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:24,647 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:25,655 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:25,825 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:56:25,826 DEBUG SenderThread:1752284 [sender.py:send():379] send: files +2024-12-31 21:56:25,826 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_1d43c0fcec8d2ec15ec2.mp4 with policy now +2024-12-31 21:56:25,826 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:56:25,827 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:56:25,827 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:56:26,068 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/yjr8mkxl-media/videos/eval/1_eval_1_1d43c0fcec8d2ec15ec2.mp4 +2024-12-31 21:56:26,223 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:26,223 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:56:26,223 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/1_eval_1_1d43c0fcec8d2ec15ec2.mp4 +2024-12-31 21:56:26,223 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval +2024-12-31 21:56:26,487 INFO wandb-upload_1:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/4ob6lh5k-media/videos/eval/1_eval_1_1d43c0fcec8d2ec15ec2.mp4 +2024-12-31 21:56:26,665 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:27,664 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:28,224 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:28,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:56:28,639 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:56:28,681 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:28,800 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:29,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:30,225 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:30,664 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:31,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:32,226 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:32,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:33,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:34,227 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:34,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:34,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:35,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:36,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:37,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:38,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:39,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:40,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:40,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:41,669 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:42,665 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:43,022 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:56:43,022 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:56:43,022 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:56:43,230 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:56:43,230 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:56:43,303 DEBUG SenderThread:1752284 [sender.py:send():379] send: stats +2024-12-31 21:56:43,480 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/vmtgs6ni-events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:56:43,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:56:43,638 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:56:43,667 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:44,231 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:44,666 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:45,665 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:45,779 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:46,231 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:46,666 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:47,665 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:48,232 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:48,665 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:49,666 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:50,666 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:51,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:51,666 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:52,666 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:53,666 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:54,666 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:55,666 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:56,640 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:56,667 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:57,667 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:58,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:56:58,638 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:56:58,667 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:59,667 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:00,236 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:00,668 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:00,893 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:57:00,894 DEBUG SenderThread:1752284 [sender.py:send():379] send: files +2024-12-31 21:57:00,894 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c9be307d2b8f293d1a04.mp4 with policy now +2024-12-31 21:57:00,894 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:57:00,895 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:57:00,895 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:57:01,136 INFO wandb-upload_1:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/jt4pzupy-media/videos/eval/2_eval_2_c9be307d2b8f293d1a04.mp4 +2024-12-31 21:57:01,236 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:57:01,237 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/2_eval_2_c9be307d2b8f293d1a04.mp4 +2024-12-31 21:57:01,237 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval +2024-12-31 21:57:01,506 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/o23kk6if-media/videos/eval/2_eval_2_c9be307d2b8f293d1a04.mp4 +2024-12-31 21:57:01,678 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:01,914 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:02,237 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:02,672 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:03,675 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:04,687 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:05,675 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:06,238 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:06,681 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:07,308 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:07,675 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:08,239 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:08,678 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:09,676 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:10,679 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:11,678 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:12,644 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:12,678 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:13,303 DEBUG SenderThread:1752284 [sender.py:send():379] send: stats +2024-12-31 21:57:13,646 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:57:13,646 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:57:13,693 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:14,709 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:14,840 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:57:14,841 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:57:14,841 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:57:15,241 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:57:15,241 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:57:15,682 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:16,242 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:16,683 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:17,683 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:17,761 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:18,242 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:18,689 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:19,681 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:20,243 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:20,681 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:21,681 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:22,681 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:23,642 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:23,681 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:24,686 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:25,685 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:26,685 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:27,685 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:28,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:57:28,638 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:57:28,686 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:28,773 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:29,686 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:30,686 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:31,693 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:32,614 DEBUG SenderThread:1752284 [sender.py:send():379] send: files +2024-12-31 21:57:32,614 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_db8eec7ff63d941b6e62.mp4 with policy now +2024-12-31 21:57:32,620 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:57:32,621 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:57:32,621 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:57:32,621 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:57:32,693 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:32,845 INFO wandb-upload_1:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/ageg6vag-media/videos/eval/3_eval_3_db8eec7ff63d941b6e62.mp4 +2024-12-31 21:57:33,247 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:57:33,248 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/3_eval_3_db8eec7ff63d941b6e62.mp4 +2024-12-31 21:57:33,248 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval +2024-12-31 21:57:33,523 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/sp56ejdx-media/videos/eval/3_eval_3_db8eec7ff63d941b6e62.mp4 +2024-12-31 21:57:33,704 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:34,248 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:34,652 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:34,697 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:35,709 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:36,701 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:37,705 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:38,249 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:38,700 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:39,703 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:40,021 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:40,250 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:40,706 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:41,704 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:42,707 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:43,303 DEBUG SenderThread:1752284 [sender.py:send():379] send: stats +2024-12-31 21:57:43,642 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:57:43,642 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:57:43,705 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:44,712 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:45,705 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:45,761 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:46,712 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:47,711 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:47,882 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:57:47,882 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:57:47,882 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:57:48,252 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:48,253 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:57:48,253 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:57:48,523 INFO wandb-upload_1:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/b94eo6lt-events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:57:48,713 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:49,717 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:50,253 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:50,722 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:51,210 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:51,714 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:52,254 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:52,714 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:53,714 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:54,255 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:54,714 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:55,714 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:56,256 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:56,365 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:56,715 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:57,714 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:58,257 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:58,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:57:58,638 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:57:58,716 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:59,715 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:00,257 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:00,715 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:01,715 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:01,800 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:58:02,715 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:03,716 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:04,753 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:05,724 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:06,259 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:06,315 DEBUG SenderThread:1752284 [sender.py:send():379] send: files +2024-12-31 21:58:06,316 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_2da638a425088a67de89.mp4 with policy now +2024-12-31 21:58:06,318 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:58:06,318 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:58:06,319 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:06,319 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:06,560 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/nm9y1b0b-media/videos/eval/4_eval_4_2da638a425088a67de89.mp4 +2024-12-31 21:58:06,717 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:07,260 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:58:07,260 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/4_eval_4_2da638a425088a67de89.mp4 +2024-12-31 21:58:07,260 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval +2024-12-31 21:58:07,381 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:58:07,550 INFO wandb-upload_1:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/bamomg55-media/videos/eval/4_eval_4_2da638a425088a67de89.mp4 +2024-12-31 21:58:07,717 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:08,260 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:08,719 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:09,721 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:10,728 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:11,722 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:12,261 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:12,712 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:58:12,753 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:13,304 DEBUG SenderThread:1752284 [sender.py:send():379] send: stats +2024-12-31 21:58:13,646 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:58:13,646 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:58:13,734 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:14,262 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:14,737 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:15,733 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:16,730 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:17,733 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:17,735 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:58:18,738 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:19,744 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:20,734 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:21,737 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:22,738 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:22,924 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:58:22,925 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:22,925 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:58:22,943 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:23,041 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 21:58:23,041 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 21:58:23,265 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:58:23,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:58:23,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 21:58:23,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media +2024-12-31 21:58:23,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table/eval +2024-12-31 21:58:23,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table +2024-12-31 21:58:23,270 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:58:23,463 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/2eip5lf1-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 21:58:23,580 INFO wandb-upload_1:1752284 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpbsscpadf +2024-12-31 21:58:23,689 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 21:58:23,699 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:58:23,736 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:23,967 INFO SenderThread:1752284 [sender.py:send_request_log_artifact():1525] logged artifact run-bufkm1dk-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE0NDQ0Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MDU5MA==', 'latestArtifact': None}} +2024-12-31 21:58:23,967 DEBUG SenderThread:1752284 [sender.py:send():379] send: files +2024-12-31 21:58:23,967 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2024-12-31 21:58:23,967 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:58:23,967 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:23,973 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:23,974 DEBUG SenderThread:1752284 [sender.py:send():379] send: telemetry +2024-12-31 21:58:23,974 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:23,980 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:23,981 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:23,987 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:23,987 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:23,993 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:23,993 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:24,002 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:24,003 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:24,015 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:24,015 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:24,027 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:24,027 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:24,039 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:24,040 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:24,051 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:24,051 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:24,063 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:24,064 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:24,072 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:24,072 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:24,081 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:24,081 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 21:58:24,226 INFO wandb-upload_2:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/e22akt3j-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 21:58:24,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:24,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:58:24,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:58:24,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json +2024-12-31 21:58:24,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table/eval +2024-12-31 21:58:24,456 INFO wandb-upload_1:1752284 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpy2wj9vf2 +2024-12-31 21:58:24,485 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/h0kj7pdu-media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json +2024-12-31 21:58:24,735 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:24,941 INFO SenderThread:1752284 [sender.py:send_request_log_artifact():1525] logged artifact run-bufkm1dk-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE0NDQ1Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MDU5Mg==', 'latestArtifact': None}} +2024-12-31 21:58:24,941 DEBUG SenderThread:1752284 [sender.py:send():379] send: files +2024-12-31 21:58:24,941 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json with policy now +2024-12-31 21:58:24,942 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:58:24,942 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:24,942 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:25,199 INFO wandb-upload_2:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/3n6co8e9-media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json +2024-12-31 21:58:25,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:58:25,735 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:26,267 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:26,736 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:27,736 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:27,946 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:58:28,222 DEBUG SenderThread:1752284 [sender.py:send():379] send: exit +2024-12-31 21:58:28,222 INFO SenderThread:1752284 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 21:58:28,222 INFO SenderThread:1752284 [sender.py:send_exit():588] handling runtime: 194 +2024-12-31 21:58:28,222 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:28,222 INFO SenderThread:1752284 [sender.py:send_exit():594] send defer +2024-12-31 21:58:28,222 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:28,222 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 21:58:28,223 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:28,223 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 21:58:28,223 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 1 +2024-12-31 21:58:28,223 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:28,223 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 21:58:28,223 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:28,223 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 21:58:28,223 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 2 +2024-12-31 21:58:28,223 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:28,223 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 21:58:28,223 INFO HandlerThread:1752284 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 21:58:28,223 DEBUG SystemMonitor:1752284 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 21:58:28,223 DEBUG SystemMonitor:1752284 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 21:58:28,224 INFO HandlerThread:1752284 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 21:58:28,224 INFO HandlerThread:1752284 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 21:58:28,268 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/config.yaml +2024-12-31 21:58:28,268 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:58:28,275 INFO HandlerThread:1752284 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 21:58:28,276 INFO HandlerThread:1752284 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 21:58:28,276 INFO HandlerThread:1752284 [interfaces.py:finish():200] Joined network monitor +2024-12-31 21:58:28,276 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:28,276 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 21:58:28,276 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 3 +2024-12-31 21:58:28,276 DEBUG SenderThread:1752284 [sender.py:send():379] send: stats +2024-12-31 21:58:28,276 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:28,277 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 21:58:28,277 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:28,277 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 21:58:28,277 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 4 +2024-12-31 21:58:28,277 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:28,277 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 21:58:30,269 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:34,931 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:58:34,932 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:34,932 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 21:58:34,932 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 5 +2024-12-31 21:58:34,932 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:58:34,932 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:58:34,932 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 21:58:34,933 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:34,933 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 21:58:34,934 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:58:34,934 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:34,935 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:34,936 DEBUG SenderThread:1752284 [sender.py:send():379] send: summary +2024-12-31 21:58:34,938 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:34,938 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:34,938 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 21:58:34,938 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 6 +2024-12-31 21:58:34,938 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:34,938 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 21:58:34,938 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:34,938 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 21:58:34,938 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 7 +2024-12-31 21:58:34,938 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:58:34,938 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:34,938 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 21:58:34,939 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:34,939 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 21:58:35,223 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:58:35,271 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:58:36,050 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 8 +2024-12-31 21:58:36,050 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:58:36,050 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:36,050 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 21:58:36,050 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:36,050 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 21:58:36,051 INFO SenderThread:1752284 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 21:58:36,053 INFO SenderThread:1752284 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 21:58:36,055 INFO SenderThread:1752284 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 21:58:36,068 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 9 +2024-12-31 21:58:36,068 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:36,069 DEBUG SenderThread:1752284 [sender.py:send():379] send: artifact +2024-12-31 21:58:36,069 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 21:58:36,223 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:58:36,271 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:36,287 INFO SenderThread:1752284 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDE0MjIyOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDE0MjIyOA==', 'versionIndex': 394}}} +2024-12-31 21:58:36,288 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:36,288 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 21:58:36,288 INFO SenderThread:1752284 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 21:58:37,272 INFO SenderThread:1752284 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files +2024-12-31 21:58:37,272 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/config.yaml config.yaml +2024-12-31 21:58:37,272 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json wandb-summary.json +2024-12-31 21:58:37,272 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/requirements.txt requirements.txt +2024-12-31 21:58:37,272 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-metadata.json wandb-metadata.json +2024-12-31 21:58:37,272 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:58:37,273 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log output.log +2024-12-31 21:58:37,273 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/diff.patch diff.patch +2024-12-31 21:58:37,277 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 21:58:37,279 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/1_eval_1_1d43c0fcec8d2ec15ec2.mp4 media/videos/eval/1_eval_1_1d43c0fcec8d2ec15ec2.mp4 +2024-12-31 21:58:37,279 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/0_eval_0_4df030dc74284b36be4e.mp4 media/videos/eval/0_eval_0_4df030dc74284b36be4e.mp4 +2024-12-31 21:58:37,279 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/2_eval_2_c9be307d2b8f293d1a04.mp4 media/videos/eval/2_eval_2_c9be307d2b8f293d1a04.mp4 +2024-12-31 21:58:37,280 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/3_eval_3_db8eec7ff63d941b6e62.mp4 media/videos/eval/3_eval_3_db8eec7ff63d941b6e62.mp4 +2024-12-31 21:58:37,281 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/4_eval_4_2da638a425088a67de89.mp4 media/videos/eval/4_eval_4_2da638a425088a67de89.mp4 +2024-12-31 21:58:37,281 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json +2024-12-31 21:58:37,281 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 21:58:37,281 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 10 +2024-12-31 21:58:37,281 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:58:37,281 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:37,281 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 21:58:37,281 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:37,281 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 21:58:37,282 INFO SenderThread:1752284 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:58:37,461 INFO wandb-upload_1:1752284 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:58:37,490 INFO wandb-upload_2:1752284 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/requirements.txt +2024-12-31 21:58:37,499 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/config.yaml +2024-12-31 21:58:37,533 INFO wandb-upload_4:1752284 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:37,595 INFO wandb-upload_3:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/8xkhmaun-events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:58:37,796 INFO Thread-11 (_thread_body):1752284 [sender.py:transition_state():614] send defer: 11 +2024-12-31 21:58:37,796 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:37,796 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 21:58:37,796 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:37,796 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 21:58:37,796 INFO SenderThread:1752284 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 21:58:37,797 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 12 +2024-12-31 21:58:37,797 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:37,798 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 21:58:37,798 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:37,798 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 21:58:37,798 INFO SenderThread:1752284 [file_stream.py:finish():601] file stream finish called +2024-12-31 21:58:37,965 INFO SenderThread:1752284 [file_stream.py:finish():605] file stream finish is done +2024-12-31 21:58:37,965 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 13 +2024-12-31 21:58:37,965 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:37,965 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 21:58:37,965 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:37,965 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 21:58:37,965 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 14 +2024-12-31 21:58:37,966 DEBUG SenderThread:1752284 [sender.py:send():379] send: final +2024-12-31 21:58:37,966 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:37,966 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 21:58:37,966 DEBUG SenderThread:1752284 [sender.py:send():379] send: footer +2024-12-31 21:58:37,966 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:37,966 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 21:58:37,966 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:58:37,967 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:58:37,967 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 21:58:37,967 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: server_info +2024-12-31 21:58:37,968 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:58:37,969 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 21:58:37,969 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 21:58:37,970 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:38,020 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:58:38,020 INFO MainThread:1752284 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 21:58:38,020 INFO MainThread:1752284 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 21:58:38,021 INFO MainThread:1752284 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 21:58:38,021 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 21:58:38,021 INFO HandlerThread:1752284 [handler.py:finish():882] shutting down handler +2024-12-31 21:58:38,969 INFO WriterThread:1752284 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/run-bufkm1dk.wandb +2024-12-31 21:58:39,020 INFO SenderThread:1752284 [sender.py:finish():1615] shutting down sender +2024-12-31 21:58:39,020 INFO SenderThread:1752284 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:58:39,020 INFO SenderThread:1752284 [file_pusher.py:join():175] waiting for file pusher diff --git a/bufkm1dk/wandb/debug.log b/bufkm1dk/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b5c87d690a715f60759efdebe829b150ed93d934 --- /dev/null +++ b/bufkm1dk/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_setup.py:_flush():76] Configure stats pid to 1752108 +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/settings +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/logs/debug.log +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/logs/debug-internal.log +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_init.py:init():569] calling init triggers +2024-12-31 21:55:12,769 INFO MainThread:1752108 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk'} +2024-12-31 21:55:12,769 INFO MainThread:1752108 [wandb_init.py:init():619] starting backend +2024-12-31 21:55:12,769 INFO MainThread:1752108 [wandb_init.py:init():623] setting up manager +2024-12-31 21:55:12,773 INFO MainThread:1752108 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 21:55:12,774 INFO MainThread:1752108 [wandb_init.py:init():631] backend started and connected +2024-12-31 21:55:12,789 INFO MainThread:1752108 [wandb_init.py:init():720] updated telemetry +2024-12-31 21:55:12,795 INFO MainThread:1752108 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 21:55:13,217 INFO MainThread:1752108 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 21:55:13,288 INFO MainThread:1752108 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 21:55:13,288 INFO MainThread:1752108 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 21:55:13,635 INFO MainThread:1752108 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 21:55:13,636 INFO MainThread:1752108 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 21:55:13,636 INFO MainThread:1752108 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 21:55:13,636 INFO MainThread:1752108 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 21:55:13,637 INFO MainThread:1752108 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 21:55:13,638 INFO MainThread:1752108 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/tb, True +2024-12-31 21:55:20,039 INFO MainThread:1752108 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2024-12-31 21:58:39,022 WARNING MsgRouterThr:1752108 [router.py:message_loop():77] message_loop has been closed diff --git a/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/code/FISH/eval_robot.py b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..41791869dc3657b820d72a52a39ee14580eb36bd --- /dev/null +++ b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + "config.grasped_obj_name": grasped_obj_name, + "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + total_num_episodes += filtered_run.summary_metrics['episode'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/config.yaml b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d4849d1fb599e656c0b241d3798ead472ac601d --- /dev/null +++ b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1745_0' +wandb_notes: + desc: null + value: 1745_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735700112 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/diff.patch b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..803862adc5267428c9ad987aaa26b0cf28ac9c13 --- /dev/null +++ b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/diff.patch @@ -0,0 +1,54 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..57a851f 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1007_0' ++# wandb_run_id: '1002_0' + ++# all books, 6/20 demos per book ++# RGBD+mask+act history ++# wandb_run_id: '1007_0' ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): diff --git a/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json new file mode 100644 index 0000000000000000000000000000000000000000..e69fc5d03f73383d42d59221f6b82c19b47fac4f --- /dev/null +++ b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[0, 10, 0.0, 0, 0.3317304849961171, 0, 0.3041884303484581, 0, 0.1718106568516029, 0, 0.19463173306135229]]} \ No newline at end of file diff --git a/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/0_eval_0_4df030dc74284b36be4e.mp4 b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/0_eval_0_4df030dc74284b36be4e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7f9f9dc563ef3eeafe6cedf852d72b76a5434eb9 --- /dev/null +++ b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/0_eval_0_4df030dc74284b36be4e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4df030dc74284b36be4e4a4c34dde9440321e9ec9fb2195051a8468e1c3f90c8 +size 659178 diff --git a/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/1_eval_1_1d43c0fcec8d2ec15ec2.mp4 b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/1_eval_1_1d43c0fcec8d2ec15ec2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..af119e619c8883a98d12ed7ae9a4fb70215aa80d --- /dev/null +++ b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/1_eval_1_1d43c0fcec8d2ec15ec2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d43c0fcec8d2ec15ec24dc9e91446c61e517a8ca1dc162cd788f815862f3d2f +size 592382 diff --git a/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/2_eval_2_c9be307d2b8f293d1a04.mp4 b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/2_eval_2_c9be307d2b8f293d1a04.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c7580cd492561b038b1cc8a638c696244eb18582 --- /dev/null +++ b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/2_eval_2_c9be307d2b8f293d1a04.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9be307d2b8f293d1a04293215598d5c3e8749d52c6526647f95f8bb87b4b746 +size 610659 diff --git a/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/3_eval_3_db8eec7ff63d941b6e62.mp4 b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/3_eval_3_db8eec7ff63d941b6e62.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..40c08a1ca28828c524b3d6439e3643f5827a55e8 --- /dev/null +++ b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/3_eval_3_db8eec7ff63d941b6e62.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db8eec7ff63d941b6e62d54aec062994dfd9909882e1b34f5b904dcdf29a14db +size 601924 diff --git a/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/4_eval_4_2da638a425088a67de89.mp4 b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/4_eval_4_2da638a425088a67de89.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5893049559bcd809857b99ab25da1b90474c7b5e --- /dev/null +++ b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/4_eval_4_2da638a425088a67de89.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2da638a425088a67de89709460e61194fdc7ea1bfb52bf3e1bcc12dd05f07021 +size 620944 diff --git a/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..d0cdc830607a3db24b79ef5a2183d820fe28e6c1 --- /dev/null +++ b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log @@ -0,0 +1,278 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735700120.039853]: resetting environment +[INFO] [1735700120.046708]: cleared current plan +[INFO] [1735700120.047557]: moving to home +[INFO] [1735700121.049337]: reached home +[INFO] [1735700121.050953]: reset action history +[INFO] [1735700123.761134]: environment reset +Starting episode 0 +[INFO] [1735700123.765342]: resetting environment +[INFO] [1735700123.769288]: cleared current plan +[INFO] [1735700123.770099]: moving to home +[INFO] [1735700124.773633]: reached home +[INFO] [1735700124.774876]: reset action history +[INFO] [1735700127.483963]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1735700132.247546]: resetting environment +[INFO] [1735700132.252446]: cleared current plan +[INFO] [1735700132.253300]: moving to home +[INFO] [1735700133.255401]: reached home +[INFO] [1735700133.256760]: reset action history +[INFO] [1735700135.969638]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/episode_rosbags/episode_0_2024-12-31-21-55-35.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1752108.log +started roslaunch server http://158.130.50.37:39879/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1752389] +process[print_text-2]: started with pid [1752413] +process[rosbag_record-3]: started with pid [1752414] +started bagging! +preempting episode +[INFO] [1735700150.506043]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1735700151.279176]: Stored episode 1. +[INFO] [1735700151.279405]: Saving video... +[INFO] [1735700151.481474]: Video saved! +Starting episode 1 +[INFO] [1735700151.501334]: resetting environment +[INFO] [1735700151.511584]: cleared current plan +[INFO] [1735700151.513599]: moving to home +[INFO] [1735700154.816210]: reached home +[INFO] [1735700154.816964]: reset action history +[INFO] [1735700157.527911]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735700168.091075]: resetting environment +[INFO] [1735700168.094373]: cleared current plan +[INFO] [1735700168.094849]: moving to home +[INFO] [1735700169.096275]: reached home +[INFO] [1735700169.096692]: reset action history +[INFO] [1735700171.806927]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/episode_rosbags/episode_1_2024-12-31-21-56-11.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1752108.log +started roslaunch server http://158.130.50.37:36349/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1752538] +process[print_text-5]: started with pid [1752539] +process[rosbag_record-6]: started with pid [1752563] +started bagging! +For topic gripper_width: timestamp difference is 103146589 for nearest: 1735700173620015657 - target: 1735700173516869068 at idx 23 +For topic gripper_width: timestamp difference is 102135264 for nearest: 1735700177220014893 - target: 1735700177117879629 at idx 69 +preempting episode +[INFO] [1735700184.788218]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735700185.502949]: Stored episode 2. +[INFO] [1735700185.503167]: Saving video... +[INFO] [1735700185.777678]: Video saved! +Starting episode 2 +[INFO] [1735700185.809371]: resetting environment +[INFO] [1735700185.819164]: cleared current plan +[INFO] [1735700185.819781]: moving to home +[INFO] [1735700189.324012]: reached home +[INFO] [1735700189.325178]: reset action history +[INFO] [1735700192.102930]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735700202.486377]: resetting environment +[INFO] [1735700202.493907]: cleared current plan +[INFO] [1735700202.494102]: moving to home +[INFO] [1735700203.495520]: reached home +[INFO] [1735700203.495952]: reset action history +[INFO] [1735700206.206594]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/episode_rosbags/episode_2_2024-12-31-21-56-46.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1752108.log +started roslaunch server http://158.130.50.37:35923/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1752701] +process[print_text-8]: started with pid [1752702] +process[rosbag_record-9]: started with pid [1752703] +started bagging! +For topic gripper_width: timestamp difference is 105644789 for nearest: 1735700218053341713 - target: 1735700217947696924 at idx 154 +preempting episode +[INFO] [1735700219.848931]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1735700220.584299]: Stored episode 3. +[INFO] [1735700220.588637]: Saving video... +[INFO] [1735700220.850475]: Video saved! +Starting episode 3 +[INFO] [1735700220.883923]: resetting environment +[INFO] [1735700220.913966]: cleared current plan +[INFO] [1735700220.914268]: moving to home +[INFO] [1735700224.417360]: reached home +[INFO] [1735700224.422889]: reset action history +[INFO] [1735700227.306742]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735700234.692483]: resetting environment +[INFO] [1735700234.708815]: cleared current plan +[INFO] [1735700234.711977]: moving to home +[INFO] [1735700235.715779]: reached home +[INFO] [1735700235.717007]: reset action history +[INFO] [1735700238.489545]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/episode_rosbags/episode_3_2024-12-31-21-57-18.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1752108.log +started roslaunch server http://158.130.50.37:44229/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1752817] +process[print_text-11]: started with pid [1752818] +process[rosbag_record-12]: started with pid [1752842] +started bagging! +preempting episode +[INFO] [1735700251.498312]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1735700252.347095]: Stored episode 4. +[INFO] [1735700252.354092]: Saving video... +[INFO] [1735700252.573699]: Video saved! +Starting episode 4 +[INFO] [1735700252.625584]: resetting environment +[INFO] [1735700252.654558]: cleared current plan +[INFO] [1735700252.654819]: moving to home +[INFO] [1735700256.162683]: reached home +[INFO] [1735700256.175118]: reset action history +[INFO] [1735700258.974026]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735700267.434495]: resetting environment +[INFO] [1735700267.449627]: cleared current plan +[INFO] [1735700267.453368]: moving to home +[INFO] [1735700268.465711]: reached home +[INFO] [1735700268.469521]: reset action history +[INFO] [1735700271.209482]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/episode_rosbags/episode_4_2024-12-31-21-57-51.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1752108.log +started roslaunch server http://158.130.50.37:46631/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1752959] +process[print_text-14]: started with pid [1752960] +process[rosbag_record-15]: started with pid [1752984] +started bagging! +For topic gripper_width: timestamp difference is 118599769 for nearest: 1735700275320045587 - target: 1735700275201445818 at idx 64 +For topic gripper_width: timestamp difference is 119603422 for nearest: 1735700276620008495 - target: 1735700276500405073 at idx 83 +For topic gripper_width: timestamp difference is 120695654 for nearest: 1735700277920041863 - target: 1735700277799346209 at idx 97 +preempting episode +[INFO] [1735700284.717482]: Storing episode... +[rosbag_record-15] killing on exit[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735700285.611441]: Stored episode 5. +[INFO] [1735700285.618931]: Saving video... +[INFO] [1735700286.256504]: Video saved! +[INFO] [1735700286.318496]: resetting environment +[INFO] [1735700286.375638]: cleared current plan +[INFO] [1735700286.380956]: moving to home +[WARN] [1735700286.004009]: Plan exhausted +[WARN] [1735700286.042230]: Plan exhausted +[WARN] [1735700286.083606]: Plan exhausted +[WARN] [1735700286.124299]: Plan exhausted +[WARN] [1735700286.165097]: Plan exhausted +[WARN] [1735700286.202547]: Plan exhausted +[WARN] [1735700286.244426]: Plan exhausted +[WARN] [1735700286.285295]: Plan exhausted +[WARN] [1735700286.324491]: Plan exhausted +[WARN] [1735700286.361707]: Plan exhausted +[INFO] [1735700289.892065]: reached home +[INFO] [1735700289.896620]: reset action history +[INFO] [1735700292.709747]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/requirements.txt b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-metadata.json b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c7ab79a46b4bb0897e640e58a84cec6701ba8009 --- /dev/null +++ b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T02:55:13.311205", + "startedAt": "2025-01-01T02:55:12.762423", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3748.3635000000004, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4492.45, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.003, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 530.4097137451172 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9f0d921ab77af03b3ff9c8e940cded138a588dbb --- /dev/null +++ b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "4df030dc74284b36be4e4a4c34dde9440321e9ec9fb2195051a8468e1c3f90c8", "size": 659178, "path": "media/videos/eval/0_eval_0_4df030dc74284b36be4e.mp4"}, "global_step": 253, "_timestamp": 1735700303.533725, "_runtime": 190.75879311561584, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "1d43c0fcec8d2ec15ec24dc9e91446c61e517a8ca1dc162cd788f815862f3d2f", "size": 592382, "path": "media/videos/eval/1_eval_1_1d43c0fcec8d2ec15ec2.mp4"}, "eval/num_success": 0.0, "episode": 10.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "c9be307d2b8f293d1a04293215598d5c3e8749d52c6526647f95f8bb87b4b746", "size": 610659, "path": "media/videos/eval/2_eval_2_c9be307d2b8f293d1a04.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "db8eec7ff63d941b6e62d54aec062994dfd9909882e1b34f5b904dcdf29a14db", "size": 601924, "path": "media/videos/eval/3_eval_3_db8eec7ff63d941b6e62.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "2da638a425088a67de89709460e61194fdc7ea1bfb52bf3e1bcc12dd05f07021", "size": 620944, "path": "media/videos/eval/4_eval_4_2da638a425088a67de89.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://xtm41by8tyqaq5i7iaj4rk7hflbdsww0wzlcdco3bsokmivnw97h259edwxemr5tm4wt1pe48kj5094mfhj76qcfmsgr81qwmbvpj8atxinr8v4dydu5eofjh6pone72/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://61h6hq2a6gmbsgj2oz5ybzf3q5fzuvobbizkaej7s85lvt1xgkrrie2obu0fz3yr1ambgm0jx368uhbi3ey9smxabq7lr22iwkkuelsd8idpzs2ipgz7yezfjijt6r9y:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 0, "total_num_episodes": 10, "total_success_rate": 0.0, "total_success_rate_lb_0.01": 0, "total_success_rate_ub_0.01": 0.3317304849961171, "total_success_rate_lb_0.025": 0, "total_success_rate_ub_0.025": 0.3041884303484581, "total_success_rate_lb_0.05": 0, "total_success_rate_ub_0.05": 0.1718106568516029, "total_success_rate_lb_0.1": 0, "total_success_rate_ub_0.1": 0.19463173306135229, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "99c5ea6d40a5d334ec51de757bf057a0ba78c69d3b5b7117fa337799eca330dd", "size": 437, "artifact_path": "wandb-client-artifact://wctptc4p0o3fssj83254ovnecwycna06n7azj2htj5q6y3d3ljeut49db8xfrt7e1b8hi8ycdm7zxupc6j8d57nt90hwpax45274fer78168j5npjoh3uscity88838n/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://hh1kc955nq8ziex45zi3dcpneu3upj1oyzrpr7o00x4n788huxx5avzajfglr2uy4w4x844ocyyo20zgn31q59o8l7c93xxtfbamjb5hk2658opadakvzupkqkjz6bgt:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 194}, "eval/total_success_rate": 0.0, "eval/total_success_rate_lb_0.01": 0.0, "eval/total_success_rate_ub_0.01": 0.3317304849624634, "eval/total_success_rate_lb_0.025": 0.0, "eval/total_success_rate_ub_0.025": 0.30418843030929565, "eval/total_success_rate_lb_0.05": 0.0, "eval/total_success_rate_ub_0.05": 0.17181065678596497, "eval/total_success_rate_lb_0.1": 0.0, "eval/total_success_rate_ub_0.1": 0.19463174045085907} \ No newline at end of file diff --git a/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/logs/debug-internal.log b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..95cb8fead48f5a61b6f023cd3d48231b20849487 --- /dev/null +++ b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/logs/debug-internal.log @@ -0,0 +1,671 @@ +2024-12-31 21:55:12,775 INFO StreamThr :1752284 [internal.py:wandb_internal():85] W&B internal server running at pid: 1752284, started at: 2024-12-31 21:55:12.774562 +2024-12-31 21:55:12,775 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status +2024-12-31 21:55:12,789 INFO WriterThread:1752284 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/run-bufkm1dk.wandb +2024-12-31 21:55:12,790 DEBUG SenderThread:1752284 [sender.py:send():379] send: header +2024-12-31 21:55:12,798 DEBUG SenderThread:1752284 [sender.py:send():379] send: run +2024-12-31 21:55:13,200 INFO SenderThread:1752284 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files +2024-12-31 21:55:13,200 INFO SenderThread:1752284 [sender.py:_start_run_threads():1188] run started: bufkm1dk with start time 1735700112.774932 +2024-12-31 21:55:13,217 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 21:55:13,218 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: check_version +2024-12-31 21:55:13,293 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 21:55:13,298 DEBUG HandlerThread:1752284 [system_info.py:__init__():26] System info init +2024-12-31 21:55:13,298 DEBUG HandlerThread:1752284 [system_info.py:__init__():41] System info init done +2024-12-31 21:55:13,298 INFO HandlerThread:1752284 [system_monitor.py:start():194] Starting system monitor +2024-12-31 21:55:13,298 INFO SystemMonitor:1752284 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 21:55:13,298 INFO HandlerThread:1752284 [system_monitor.py:probe():214] Collecting system info +2024-12-31 21:55:13,299 INFO SystemMonitor:1752284 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 21:55:13,299 INFO SystemMonitor:1752284 [interfaces.py:start():188] Started disk monitoring +2024-12-31 21:55:13,300 INFO SystemMonitor:1752284 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 21:55:13,301 INFO SystemMonitor:1752284 [interfaces.py:start():188] Started memory monitoring +2024-12-31 21:55:13,301 INFO SystemMonitor:1752284 [interfaces.py:start():188] Started network monitoring +2024-12-31 21:55:13,311 DEBUG HandlerThread:1752284 [system_info.py:probe():152] Probing system +2024-12-31 21:55:13,312 DEBUG HandlerThread:1752284 [system_info.py:_probe_git():137] Probing git +2024-12-31 21:55:13,318 DEBUG HandlerThread:1752284 [system_info.py:_probe_git():145] Probing git done +2024-12-31 21:55:13,318 DEBUG HandlerThread:1752284 [system_info.py:probe():200] Probing system done +2024-12-31 21:55:13,318 DEBUG HandlerThread:1752284 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T02:55:13.311205', 'startedAt': '2025-01-01T02:55:12.762423', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.3635000000004, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4492.45, 'min': 800.0, 'max': 4700.0}, {'current': 4500.003, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 530.4097137451172}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 21:55:13,319 INFO HandlerThread:1752284 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 21:55:13,319 INFO HandlerThread:1752284 [system_monitor.py:probe():227] Publishing system info +2024-12-31 21:55:13,319 DEBUG HandlerThread:1752284 [system_info.py:_save_code():44] Saving code +2024-12-31 21:55:13,320 DEBUG HandlerThread:1752284 [system_info.py:_save_code():67] Saving code done +2024-12-31 21:55:13,320 DEBUG HandlerThread:1752284 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 21:55:13,384 DEBUG HandlerThread:1752284 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 21:55:13,384 INFO HandlerThread:1752284 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 21:55:13,387 DEBUG SenderThread:1752284 [sender.py:send():379] send: files +2024-12-31 21:55:13,388 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 21:55:13,388 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 21:55:13,388 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 21:55:13,635 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 21:55:13,635 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: python_packages +2024-12-31 21:55:13,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:55:13,636 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:55:13,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:13,638 INFO HandlerThread:1752284 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/tb" + save: true +} +control { + relay_id: "10baa15bb7d642b7af75b0e9ed3f9ec7" +} +_info { + stream_id: "bufkm1dk" +} + +2024-12-31 21:55:13,638 DEBUG HandlerThread:1752284 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 21:55:13,680 INFO wandb-upload_2:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/764pzyax-diff.patch +2024-12-31 21:55:13,680 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/8qfd83d2-wandb-metadata.json +2024-12-31 21:55:13,744 DEBUG SenderThread:1752284 [sender.py:send():379] send: telemetry +2024-12-31 21:55:13,744 DEBUG SenderThread:1752284 [sender.py:send():379] send: tbrecord +2024-12-31 21:55:13,745 DEBUG SenderThread:1752284 [sender.py:send():379] send: files +2024-12-31 21:55:13,745 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 with policy live +2024-12-31 21:55:13,936 INFO wandb-upload_2:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/rur3i32c-events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:55:13,994 INFO wandb-upload_1:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/lp8puosj-code/FISH/eval_robot.py +2024-12-31 21:55:14,201 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/diff.patch +2024-12-31 21:55:14,201 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:55:14,202 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/code/FISH/eval_robot.py +2024-12-31 21:55:14,202 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-metadata.json +2024-12-31 21:55:14,202 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:14,202 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/requirements.txt +2024-12-31 21:55:14,202 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/code +2024-12-31 21:55:14,202 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/code/FISH +2024-12-31 21:55:14,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:15,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:16,200 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:16,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:17,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:18,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:55:18,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:19,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:20,039 DEBUG SenderThread:1752284 [sender.py:send():379] send: config +2024-12-31 21:55:20,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:21,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:22,202 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:22,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:23,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:55:23,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:24,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:25,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:26,203 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:26,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:27,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:28,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:55:28,636 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:55:28,677 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:28,733 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:55:29,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:30,204 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:30,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:31,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:32,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:33,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:34,206 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:34,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:55:34,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:35,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:36,206 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:36,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:37,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:38,207 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:38,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:39,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:55:39,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:40,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:41,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:42,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:43,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:55:43,636 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:55:43,677 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:44,640 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:44,725 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:55:45,209 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/config.yaml +2024-12-31 21:55:45,640 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:46,640 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:47,640 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:48,640 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:49,641 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:50,503 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:55:50,641 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:51,497 DEBUG SenderThread:1752284 [sender.py:send():379] send: files +2024-12-31 21:55:51,497 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_4df030dc74284b36be4e.mp4 with policy now +2024-12-31 21:55:51,499 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:55:51,500 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:55:51,500 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:55:51,501 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:55:51,641 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:51,743 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/awawrot7-media/videos/eval/0_eval_0_4df030dc74284b36be4e.mp4 +2024-12-31 21:55:52,211 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:52,211 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/0_eval_0_4df030dc74284b36be4e.mp4 +2024-12-31 21:55:52,211 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:55:52,211 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media +2024-12-31 21:55:52,211 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos +2024-12-31 21:55:52,211 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval +2024-12-31 21:55:52,489 INFO wandb-upload_1:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/qg8ywmar-media/videos/eval/0_eval_0_4df030dc74284b36be4e.mp4 +2024-12-31 21:55:52,641 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:53,641 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:54,212 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:54,641 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:55,641 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:55,857 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:55:56,213 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:55:56,641 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:57,641 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:58,636 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:55:58,636 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:55:58,677 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:55:59,643 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:00,214 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:00,643 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:01,643 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:01,739 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:02,643 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:03,644 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:04,644 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:05,644 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:06,644 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:07,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:07,644 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:08,217 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:56:08,644 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:09,644 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:10,218 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:10,645 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:11,645 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:12,645 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:13,076 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:13,301 DEBUG SystemMonitor:1752284 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 21:56:13,302 DEBUG SenderThread:1752284 [sender.py:send():379] send: stats +2024-12-31 21:56:13,665 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:56:13,665 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:13,665 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:56:14,219 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:14,645 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:15,645 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:16,219 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:16,645 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:17,646 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:18,220 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:18,269 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:18,646 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:19,646 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:20,646 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:21,646 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:22,647 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:23,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:23,647 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:24,647 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:25,655 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:25,825 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:56:25,826 DEBUG SenderThread:1752284 [sender.py:send():379] send: files +2024-12-31 21:56:25,826 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_1d43c0fcec8d2ec15ec2.mp4 with policy now +2024-12-31 21:56:25,826 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:56:25,827 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:56:25,827 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:56:26,068 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/yjr8mkxl-media/videos/eval/1_eval_1_1d43c0fcec8d2ec15ec2.mp4 +2024-12-31 21:56:26,223 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:26,223 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:56:26,223 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/1_eval_1_1d43c0fcec8d2ec15ec2.mp4 +2024-12-31 21:56:26,223 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval +2024-12-31 21:56:26,487 INFO wandb-upload_1:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/4ob6lh5k-media/videos/eval/1_eval_1_1d43c0fcec8d2ec15ec2.mp4 +2024-12-31 21:56:26,665 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:27,664 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:28,224 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:28,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:56:28,639 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:56:28,681 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:28,800 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:29,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:30,225 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:30,664 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:31,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:32,226 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:32,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:33,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:34,227 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:34,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:34,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:35,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:36,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:37,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:38,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:39,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:40,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:40,663 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:41,669 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:42,665 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:43,022 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:56:43,022 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:56:43,022 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:56:43,230 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:56:43,230 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:56:43,303 DEBUG SenderThread:1752284 [sender.py:send():379] send: stats +2024-12-31 21:56:43,480 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/vmtgs6ni-events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:56:43,637 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:56:43,638 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:56:43,667 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:44,231 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:44,666 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:45,665 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:45,779 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:46,231 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:46,666 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:47,665 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:48,232 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:56:48,665 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:49,666 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:50,666 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:51,639 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:51,666 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:52,666 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:53,666 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:54,666 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:55,666 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:56,640 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:56:56,667 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:57,667 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:58,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:56:58,638 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:56:58,667 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:56:59,667 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:00,236 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:00,668 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:00,893 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:57:00,894 DEBUG SenderThread:1752284 [sender.py:send():379] send: files +2024-12-31 21:57:00,894 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c9be307d2b8f293d1a04.mp4 with policy now +2024-12-31 21:57:00,894 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:57:00,895 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:57:00,895 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:57:01,136 INFO wandb-upload_1:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/jt4pzupy-media/videos/eval/2_eval_2_c9be307d2b8f293d1a04.mp4 +2024-12-31 21:57:01,236 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:57:01,237 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/2_eval_2_c9be307d2b8f293d1a04.mp4 +2024-12-31 21:57:01,237 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval +2024-12-31 21:57:01,506 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/o23kk6if-media/videos/eval/2_eval_2_c9be307d2b8f293d1a04.mp4 +2024-12-31 21:57:01,678 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:01,914 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:02,237 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:02,672 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:03,675 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:04,687 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:05,675 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:06,238 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:06,681 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:07,308 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:07,675 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:08,239 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:08,678 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:09,676 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:10,679 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:11,678 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:12,644 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:12,678 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:13,303 DEBUG SenderThread:1752284 [sender.py:send():379] send: stats +2024-12-31 21:57:13,646 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:57:13,646 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:57:13,693 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:14,709 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:14,840 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:57:14,841 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:57:14,841 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:57:15,241 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:57:15,241 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:57:15,682 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:16,242 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:16,683 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:17,683 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:17,761 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:18,242 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:18,689 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:19,681 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:20,243 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:20,681 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:21,681 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:22,681 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:23,642 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:23,681 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:24,686 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:25,685 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:26,685 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:27,685 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:28,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:57:28,638 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:57:28,686 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:28,773 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:29,686 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:30,686 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:31,693 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:32,614 DEBUG SenderThread:1752284 [sender.py:send():379] send: files +2024-12-31 21:57:32,614 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_db8eec7ff63d941b6e62.mp4 with policy now +2024-12-31 21:57:32,620 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:57:32,621 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:57:32,621 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:57:32,621 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:57:32,693 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:32,845 INFO wandb-upload_1:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/ageg6vag-media/videos/eval/3_eval_3_db8eec7ff63d941b6e62.mp4 +2024-12-31 21:57:33,247 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:57:33,248 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/3_eval_3_db8eec7ff63d941b6e62.mp4 +2024-12-31 21:57:33,248 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval +2024-12-31 21:57:33,523 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/sp56ejdx-media/videos/eval/3_eval_3_db8eec7ff63d941b6e62.mp4 +2024-12-31 21:57:33,704 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:34,248 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:34,652 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:34,697 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:35,709 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:36,701 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:37,705 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:38,249 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:38,700 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:39,703 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:40,021 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:40,250 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:40,706 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:41,704 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:42,707 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:43,303 DEBUG SenderThread:1752284 [sender.py:send():379] send: stats +2024-12-31 21:57:43,642 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:57:43,642 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:57:43,705 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:44,712 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:45,705 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:45,761 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:46,712 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:47,711 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:47,882 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:57:47,882 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:57:47,882 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:57:48,252 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:48,253 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:57:48,253 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:57:48,523 INFO wandb-upload_1:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/b94eo6lt-events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:57:48,713 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:49,717 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:50,253 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:50,722 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:51,210 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:51,714 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:52,254 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:52,714 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:53,714 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:54,255 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:54,714 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:55,714 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:56,256 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:56,365 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:57:56,715 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:57,714 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:58,257 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:57:58,638 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:57:58,638 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:57:58,716 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:57:59,715 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:00,257 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:00,715 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:01,715 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:01,800 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:58:02,715 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:03,716 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:04,753 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:05,724 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:06,259 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:06,315 DEBUG SenderThread:1752284 [sender.py:send():379] send: files +2024-12-31 21:58:06,316 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_2da638a425088a67de89.mp4 with policy now +2024-12-31 21:58:06,318 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:58:06,318 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:58:06,319 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:06,319 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:06,560 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/nm9y1b0b-media/videos/eval/4_eval_4_2da638a425088a67de89.mp4 +2024-12-31 21:58:06,717 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:07,260 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:58:07,260 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/4_eval_4_2da638a425088a67de89.mp4 +2024-12-31 21:58:07,260 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval +2024-12-31 21:58:07,381 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:58:07,550 INFO wandb-upload_1:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/bamomg55-media/videos/eval/4_eval_4_2da638a425088a67de89.mp4 +2024-12-31 21:58:07,717 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:08,260 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:08,719 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:09,721 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:10,728 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:11,722 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:12,261 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:12,712 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:58:12,753 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:13,304 DEBUG SenderThread:1752284 [sender.py:send():379] send: stats +2024-12-31 21:58:13,646 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 21:58:13,646 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: stop_status +2024-12-31 21:58:13,734 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:14,262 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:14,737 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:15,733 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:16,730 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:17,733 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:17,735 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:58:18,738 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:19,744 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:20,734 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:21,737 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:22,738 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:22,924 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:58:22,925 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:22,925 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:58:22,943 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:23,041 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 21:58:23,041 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 21:58:23,265 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:58:23,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:58:23,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 21:58:23,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media +2024-12-31 21:58:23,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table/eval +2024-12-31 21:58:23,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table +2024-12-31 21:58:23,270 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:58:23,463 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/2eip5lf1-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 21:58:23,580 INFO wandb-upload_1:1752284 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpbsscpadf +2024-12-31 21:58:23,689 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 21:58:23,699 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:58:23,736 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:23,967 INFO SenderThread:1752284 [sender.py:send_request_log_artifact():1525] logged artifact run-bufkm1dk-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE0NDQ0Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MDU5MA==', 'latestArtifact': None}} +2024-12-31 21:58:23,967 DEBUG SenderThread:1752284 [sender.py:send():379] send: files +2024-12-31 21:58:23,967 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2024-12-31 21:58:23,967 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:58:23,967 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:23,973 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:23,974 DEBUG SenderThread:1752284 [sender.py:send():379] send: telemetry +2024-12-31 21:58:23,974 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:23,980 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:23,981 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:23,987 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:23,987 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:23,993 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:23,993 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:24,002 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:24,003 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:24,015 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:24,015 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:24,027 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:24,027 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:24,039 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:24,040 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:24,051 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:24,051 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:24,063 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:24,064 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:24,072 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:24,072 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:24,081 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:24,081 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 21:58:24,226 INFO wandb-upload_2:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/e22akt3j-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 21:58:24,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:24,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:58:24,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:58:24,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json +2024-12-31 21:58:24,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table/eval +2024-12-31 21:58:24,456 INFO wandb-upload_1:1752284 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpy2wj9vf2 +2024-12-31 21:58:24,485 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/h0kj7pdu-media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json +2024-12-31 21:58:24,735 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:24,941 INFO SenderThread:1752284 [sender.py:send_request_log_artifact():1525] logged artifact run-bufkm1dk-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE0NDQ1Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MDU5Mg==', 'latestArtifact': None}} +2024-12-31 21:58:24,941 DEBUG SenderThread:1752284 [sender.py:send():379] send: files +2024-12-31 21:58:24,941 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json with policy now +2024-12-31 21:58:24,942 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:58:24,942 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:24,942 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:25,199 INFO wandb-upload_2:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/3n6co8e9-media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json +2024-12-31 21:58:25,266 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:58:25,735 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:26,267 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:26,736 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:27,736 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:27,946 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:58:28,222 DEBUG SenderThread:1752284 [sender.py:send():379] send: exit +2024-12-31 21:58:28,222 INFO SenderThread:1752284 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 21:58:28,222 INFO SenderThread:1752284 [sender.py:send_exit():588] handling runtime: 194 +2024-12-31 21:58:28,222 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:28,222 INFO SenderThread:1752284 [sender.py:send_exit():594] send defer +2024-12-31 21:58:28,222 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:28,222 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 21:58:28,223 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:28,223 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 21:58:28,223 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 1 +2024-12-31 21:58:28,223 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:28,223 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 21:58:28,223 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:28,223 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 21:58:28,223 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 2 +2024-12-31 21:58:28,223 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:28,223 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 21:58:28,223 INFO HandlerThread:1752284 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 21:58:28,223 DEBUG SystemMonitor:1752284 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 21:58:28,223 DEBUG SystemMonitor:1752284 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 21:58:28,224 INFO HandlerThread:1752284 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 21:58:28,224 INFO HandlerThread:1752284 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 21:58:28,268 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/config.yaml +2024-12-31 21:58:28,268 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:58:28,275 INFO HandlerThread:1752284 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 21:58:28,276 INFO HandlerThread:1752284 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 21:58:28,276 INFO HandlerThread:1752284 [interfaces.py:finish():200] Joined network monitor +2024-12-31 21:58:28,276 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:28,276 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 21:58:28,276 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 3 +2024-12-31 21:58:28,276 DEBUG SenderThread:1752284 [sender.py:send():379] send: stats +2024-12-31 21:58:28,276 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:28,277 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 21:58:28,277 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:28,277 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 21:58:28,277 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 4 +2024-12-31 21:58:28,277 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:28,277 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 21:58:30,269 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:34,931 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:58:34,932 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:34,932 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 21:58:34,932 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 5 +2024-12-31 21:58:34,932 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:58:34,932 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:58:34,932 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 21:58:34,933 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:34,933 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 21:58:34,934 DEBUG SenderThread:1752284 [sender.py:send():379] send: history +2024-12-31 21:58:34,934 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:58:34,935 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:34,936 DEBUG SenderThread:1752284 [sender.py:send():379] send: summary +2024-12-31 21:58:34,938 INFO SenderThread:1752284 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:58:34,938 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:34,938 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 21:58:34,938 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 6 +2024-12-31 21:58:34,938 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:34,938 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 21:58:34,938 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:34,938 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 21:58:34,938 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 7 +2024-12-31 21:58:34,938 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:58:34,938 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:34,938 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 21:58:34,939 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:34,939 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 21:58:35,223 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:58:35,271 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:58:36,050 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 8 +2024-12-31 21:58:36,050 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:58:36,050 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:36,050 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 21:58:36,050 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:36,050 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 21:58:36,051 INFO SenderThread:1752284 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 21:58:36,053 INFO SenderThread:1752284 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 21:58:36,055 INFO SenderThread:1752284 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 21:58:36,068 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 9 +2024-12-31 21:58:36,068 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:36,069 DEBUG SenderThread:1752284 [sender.py:send():379] send: artifact +2024-12-31 21:58:36,069 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 21:58:36,223 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:58:36,271 INFO Thread-12 :1752284 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:36,287 INFO SenderThread:1752284 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDE0MjIyOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDE0MjIyOA==', 'versionIndex': 394}}} +2024-12-31 21:58:36,288 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:36,288 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 21:58:36,288 INFO SenderThread:1752284 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 21:58:37,272 INFO SenderThread:1752284 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files +2024-12-31 21:58:37,272 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/config.yaml config.yaml +2024-12-31 21:58:37,272 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json wandb-summary.json +2024-12-31 21:58:37,272 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/requirements.txt requirements.txt +2024-12-31 21:58:37,272 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-metadata.json wandb-metadata.json +2024-12-31 21:58:37,272 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:58:37,273 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log output.log +2024-12-31 21:58:37,273 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/diff.patch diff.patch +2024-12-31 21:58:37,277 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 21:58:37,279 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/1_eval_1_1d43c0fcec8d2ec15ec2.mp4 media/videos/eval/1_eval_1_1d43c0fcec8d2ec15ec2.mp4 +2024-12-31 21:58:37,279 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/0_eval_0_4df030dc74284b36be4e.mp4 media/videos/eval/0_eval_0_4df030dc74284b36be4e.mp4 +2024-12-31 21:58:37,279 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/2_eval_2_c9be307d2b8f293d1a04.mp4 media/videos/eval/2_eval_2_c9be307d2b8f293d1a04.mp4 +2024-12-31 21:58:37,280 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/3_eval_3_db8eec7ff63d941b6e62.mp4 media/videos/eval/3_eval_3_db8eec7ff63d941b6e62.mp4 +2024-12-31 21:58:37,281 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/videos/eval/4_eval_4_2da638a425088a67de89.mp4 media/videos/eval/4_eval_4_2da638a425088a67de89.mp4 +2024-12-31 21:58:37,281 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json +2024-12-31 21:58:37,281 INFO SenderThread:1752284 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2024-12-31 21:58:37,281 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 10 +2024-12-31 21:58:37,281 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:58:37,281 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:37,281 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 21:58:37,281 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:37,281 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 21:58:37,282 INFO SenderThread:1752284 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:58:37,461 INFO wandb-upload_1:1752284 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/wandb-summary.json +2024-12-31 21:58:37,490 INFO wandb-upload_2:1752284 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/requirements.txt +2024-12-31 21:58:37,499 INFO wandb-upload_0:1752284 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/config.yaml +2024-12-31 21:58:37,533 INFO wandb-upload_4:1752284 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/files/output.log +2024-12-31 21:58:37,595 INFO wandb-upload_3:1752284 [upload_job.py:push():130] Uploaded file /tmp/tmpkne4x1qgwandb/8xkhmaun-events.out.tfevents.1735700113.leonmkim-ROG-Strix-G15CS-G15CS.1752108.0 +2024-12-31 21:58:37,796 INFO Thread-11 (_thread_body):1752284 [sender.py:transition_state():614] send defer: 11 +2024-12-31 21:58:37,796 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:37,796 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 21:58:37,796 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:37,796 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 21:58:37,796 INFO SenderThread:1752284 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 21:58:37,797 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 12 +2024-12-31 21:58:37,797 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:37,798 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 21:58:37,798 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:37,798 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 21:58:37,798 INFO SenderThread:1752284 [file_stream.py:finish():601] file stream finish called +2024-12-31 21:58:37,965 INFO SenderThread:1752284 [file_stream.py:finish():605] file stream finish is done +2024-12-31 21:58:37,965 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 13 +2024-12-31 21:58:37,965 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:37,965 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 21:58:37,965 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:37,965 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 21:58:37,965 INFO SenderThread:1752284 [sender.py:transition_state():614] send defer: 14 +2024-12-31 21:58:37,966 DEBUG SenderThread:1752284 [sender.py:send():379] send: final +2024-12-31 21:58:37,966 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:58:37,966 INFO HandlerThread:1752284 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 21:58:37,966 DEBUG SenderThread:1752284 [sender.py:send():379] send: footer +2024-12-31 21:58:37,966 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: defer +2024-12-31 21:58:37,966 INFO SenderThread:1752284 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 21:58:37,966 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:58:37,967 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:58:37,967 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 21:58:37,967 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: server_info +2024-12-31 21:58:37,968 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:58:37,969 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 21:58:37,969 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 21:58:37,970 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:58:38,020 DEBUG SenderThread:1752284 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:58:38,020 INFO MainThread:1752284 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 21:58:38,020 INFO MainThread:1752284 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 21:58:38,021 INFO MainThread:1752284 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 21:58:38,021 DEBUG HandlerThread:1752284 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 21:58:38,021 INFO HandlerThread:1752284 [handler.py:finish():882] shutting down handler +2024-12-31 21:58:38,969 INFO WriterThread:1752284 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/run-bufkm1dk.wandb +2024-12-31 21:58:39,020 INFO SenderThread:1752284 [sender.py:finish():1615] shutting down sender +2024-12-31 21:58:39,020 INFO SenderThread:1752284 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:58:39,020 INFO SenderThread:1752284 [file_pusher.py:join():175] waiting for file pusher diff --git a/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/logs/debug.log b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b5c87d690a715f60759efdebe829b150ed93d934 --- /dev/null +++ b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_setup.py:_flush():76] Configure stats pid to 1752108 +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/settings +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/logs/debug.log +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/logs/debug-internal.log +2024-12-31 21:55:12,768 INFO MainThread:1752108 [wandb_init.py:init():569] calling init triggers +2024-12-31 21:55:12,769 INFO MainThread:1752108 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk'} +2024-12-31 21:55:12,769 INFO MainThread:1752108 [wandb_init.py:init():619] starting backend +2024-12-31 21:55:12,769 INFO MainThread:1752108 [wandb_init.py:init():623] setting up manager +2024-12-31 21:55:12,773 INFO MainThread:1752108 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 21:55:12,774 INFO MainThread:1752108 [wandb_init.py:init():631] backend started and connected +2024-12-31 21:55:12,789 INFO MainThread:1752108 [wandb_init.py:init():720] updated telemetry +2024-12-31 21:55:12,795 INFO MainThread:1752108 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 21:55:13,217 INFO MainThread:1752108 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 21:55:13,288 INFO MainThread:1752108 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 21:55:13,288 INFO MainThread:1752108 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 21:55:13,635 INFO MainThread:1752108 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 21:55:13,636 INFO MainThread:1752108 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 21:55:13,636 INFO MainThread:1752108 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 21:55:13,636 INFO MainThread:1752108 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 21:55:13,637 INFO MainThread:1752108 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 21:55:13,638 INFO MainThread:1752108 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/bufkm1dk/tb, True +2024-12-31 21:55:20,039 INFO MainThread:1752108 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2024-12-31 21:58:39,022 WARNING MsgRouterThr:1752108 [router.py:message_loop():77] message_loop has been closed diff --git a/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/run-bufkm1dk.wandb b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/run-bufkm1dk.wandb new file mode 100644 index 0000000000000000000000000000000000000000..9e7d1a60e552a7796a81bfa6d7be731dc932c9ae Binary files /dev/null and b/bufkm1dk/wandb/run-20241231_215512-bufkm1dk/run-bufkm1dk.wandb differ diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af939c87ac2ec8f12084f90f070114700992485c --- /dev/null +++ b/config.yaml @@ -0,0 +1,546 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '1745_0' +wandb_notes: 1745_0_restarted_2 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: true +use_sdf_maps: true +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 7 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/gw1x9gjs/.hydra/config.yaml b/gw1x9gjs/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57f0faa93e8c5310ed098ffd123244267c23fcf8 --- /dev/null +++ b/gw1x9gjs/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1745_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/gw1x9gjs/.hydra/hydra.yaml b/gw1x9gjs/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0480a750788d1bff2093b53f2fd26827a81d992c --- /dev/null +++ b/gw1x9gjs/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/gw1x9gjs + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/gw1x9gjs/.hydra/overrides.yaml b/gw1x9gjs/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/gw1x9gjs/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/gw1x9gjs/eval_policy.log b/gw1x9gjs/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..c70edd8834cdd3d80da8aa043e8a807bbad10d5b --- /dev/null +++ b/gw1x9gjs/eval_policy.log @@ -0,0 +1,15 @@ +[2024-12-31 23:37:41,709][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-31 23:37:41,712][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-31 23:37:44,708][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/niwmozqn/.hydra/config.yaml b/niwmozqn/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57f0faa93e8c5310ed098ffd123244267c23fcf8 --- /dev/null +++ b/niwmozqn/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1745_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/niwmozqn/.hydra/hydra.yaml b/niwmozqn/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..328805e5495cbf8b8bf0820d725d81221966672c --- /dev/null +++ b/niwmozqn/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/niwmozqn/.hydra/overrides.yaml b/niwmozqn/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/niwmozqn/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/niwmozqn/episode_rosbags/aligned_depth_to_color_K.npy b/niwmozqn/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/niwmozqn/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/niwmozqn/episode_rosbags/cam_tf_world.npy b/niwmozqn/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/niwmozqn/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/niwmozqn/episode_rosbags/color_K.npy b/niwmozqn/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/niwmozqn/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/niwmozqn/episode_rosbags/depth_K.npy b/niwmozqn/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/niwmozqn/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/niwmozqn/eval_robot.log b/niwmozqn/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..8066a257b0733b7b1daaf036867ccc515b3af35f --- /dev/null +++ b/niwmozqn/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-01 13:51:18,266][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-01 13:51:18,269][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/niwmozqn/eval_video/0_eval.mp4 b/niwmozqn/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c695b2a3050ba2f62e7796e1526f1a7d0969d29e --- /dev/null +++ b/niwmozqn/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294304c5db5cd37da633dec9ddbed006a3ffe4d2a3c0f2850acd77dfc03696cd +size 1464499 diff --git a/niwmozqn/eval_video/1_eval.mp4 b/niwmozqn/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..978272677e77d8fc2739615d8c28885b0719c18a --- /dev/null +++ b/niwmozqn/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e0e7530b69fb1fd581f0857a472524c725b947703abc37411eaf1190e11b9ea +size 1455848 diff --git a/niwmozqn/eval_video/2_eval.mp4 b/niwmozqn/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3ccda36def106f6f765aeca58c1756dab5890a62 --- /dev/null +++ b/niwmozqn/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1afdd9f2c1c0e92713f213a35100539fed551b2404536f444a52c463d048e22 +size 1449330 diff --git a/niwmozqn/eval_video/3_eval.mp4 b/niwmozqn/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..73647a39298f364bce83b8b9ddd429d1a300b828 --- /dev/null +++ b/niwmozqn/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48a28150d9236694be84fb1c4293ccbe9972c28906ca6e70f627090a7e12c862 +size 1454516 diff --git a/niwmozqn/eval_video/4_eval.mp4 b/niwmozqn/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4fa7d88f349e1be2e4eb85be1b1bb9b204ad5dda --- /dev/null +++ b/niwmozqn/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae99ac981fb1172cf28a49dfa5fa49ad6ad047d62657a8efbb3c9744f75674a6 +size 1513871 diff --git a/niwmozqn/tb/events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 b/niwmozqn/tb/events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 new file mode 100644 index 0000000000000000000000000000000000000000..3ef1a05265a3e3f9725468303cd827b67174a596 --- /dev/null +++ b/niwmozqn/tb/events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0274a5f8a3c3cc6b3c63760758e7e204d1b737614ffdc6c8b3953bacf74a5ca4 +size 6539 diff --git a/niwmozqn/wandb/debug-internal.log b/niwmozqn/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..0dc10a42d20d19c7f4e8fe12a4621daca667d0d2 --- /dev/null +++ b/niwmozqn/wandb/debug-internal.log @@ -0,0 +1,854 @@ +2025-01-01 13:51:24,033 INFO StreamThr :1808069 [internal.py:wandb_internal():85] W&B internal server running at pid: 1808069, started at: 2025-01-01 13:51:24.033568 +2025-01-01 13:51:24,034 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status +2025-01-01 13:51:24,047 INFO WriterThread:1808069 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/run-niwmozqn.wandb +2025-01-01 13:51:24,047 DEBUG SenderThread:1808069 [sender.py:send():379] send: header +2025-01-01 13:51:24,058 DEBUG SenderThread:1808069 [sender.py:send():379] send: run +2025-01-01 13:51:24,257 INFO SenderThread:1808069 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files +2025-01-01 13:51:24,257 INFO SenderThread:1808069 [sender.py:_start_run_threads():1188] run started: niwmozqn with start time 1735757484.034071 +2025-01-01 13:51:24,271 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 13:51:24,271 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: check_version +2025-01-01 13:51:24,345 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 13:51:24,350 DEBUG HandlerThread:1808069 [system_info.py:__init__():26] System info init +2025-01-01 13:51:24,350 DEBUG HandlerThread:1808069 [system_info.py:__init__():41] System info init done +2025-01-01 13:51:24,350 INFO HandlerThread:1808069 [system_monitor.py:start():194] Starting system monitor +2025-01-01 13:51:24,350 INFO SystemMonitor:1808069 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 13:51:24,351 INFO HandlerThread:1808069 [system_monitor.py:probe():214] Collecting system info +2025-01-01 13:51:24,351 INFO SystemMonitor:1808069 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 13:51:24,352 INFO SystemMonitor:1808069 [interfaces.py:start():188] Started disk monitoring +2025-01-01 13:51:24,353 INFO SystemMonitor:1808069 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 13:51:24,353 INFO SystemMonitor:1808069 [interfaces.py:start():188] Started memory monitoring +2025-01-01 13:51:24,354 INFO SystemMonitor:1808069 [interfaces.py:start():188] Started network monitoring +2025-01-01 13:51:24,363 DEBUG HandlerThread:1808069 [system_info.py:probe():152] Probing system +2025-01-01 13:51:24,364 DEBUG HandlerThread:1808069 [system_info.py:_probe_git():137] Probing git +2025-01-01 13:51:24,369 DEBUG HandlerThread:1808069 [system_info.py:_probe_git():145] Probing git done +2025-01-01 13:51:24,369 DEBUG HandlerThread:1808069 [system_info.py:probe():200] Probing system done +2025-01-01 13:51:24,369 DEBUG HandlerThread:1808069 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T18:51:24.363514', 'startedAt': '2025-01-01T18:51:24.020448', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.5651249999996, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.922, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 811.3620529174805}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 13:51:24,369 INFO HandlerThread:1808069 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 13:51:24,369 INFO HandlerThread:1808069 [system_monitor.py:probe():227] Publishing system info +2025-01-01 13:51:24,369 DEBUG HandlerThread:1808069 [system_info.py:_save_code():44] Saving code +2025-01-01 13:51:24,371 DEBUG HandlerThread:1808069 [system_info.py:_save_code():67] Saving code done +2025-01-01 13:51:24,371 DEBUG HandlerThread:1808069 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 13:51:24,432 DEBUG HandlerThread:1808069 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 13:51:24,433 INFO HandlerThread:1808069 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 13:51:24,436 DEBUG SenderThread:1808069 [sender.py:send():379] send: files +2025-01-01 13:51:24,436 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 13:51:24,436 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 13:51:24,437 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 13:51:24,681 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 13:51:24,682 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: python_packages +2025-01-01 13:51:24,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:51:24,682 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:51:24,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:24,686 INFO HandlerThread:1808069 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/tb" + save: true +} +control { + relay_id: "fc69491f6f894c9d849a333e2dc167c9" +} +_info { + stream_id: "niwmozqn" +} + +2025-01-01 13:51:24,687 DEBUG HandlerThread:1808069 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 13:51:24,687 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/ih4gd4sf-wandb-metadata.json +2025-01-01 13:51:24,707 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/t49br4cp-diff.patch +2025-01-01 13:51:24,753 DEBUG SenderThread:1808069 [sender.py:send():379] send: telemetry +2025-01-01 13:51:24,753 DEBUG SenderThread:1808069 [sender.py:send():379] send: tbrecord +2025-01-01 13:51:24,753 DEBUG SenderThread:1808069 [sender.py:send():379] send: files +2025-01-01 13:51:24,753 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 with policy live +2025-01-01 13:51:24,772 INFO wandb-upload_1:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/kpxyti30-code/FISH/eval_robot.py +2025-01-01 13:51:24,955 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/mpasxk2g-events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:51:25,258 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:25,258 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-metadata.json +2025-01-01 13:51:25,259 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/diff.patch +2025-01-01 13:51:25,259 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/requirements.txt +2025-01-01 13:51:25,259 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:51:25,259 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/code/FISH/eval_robot.py +2025-01-01 13:51:25,259 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/code/FISH +2025-01-01 13:51:25,259 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/code +2025-01-01 13:51:25,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:26,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:27,257 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:27,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:28,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:29,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:29,684 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:30,016 DEBUG SenderThread:1808069 [sender.py:send():379] send: config +2025-01-01 13:51:30,684 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:31,259 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:31,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:32,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:33,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:34,683 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:34,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:35,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:36,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:37,260 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:37,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:38,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:39,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:51:39,682 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:51:39,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:39,803 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:40,686 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:41,262 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:41,686 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:42,686 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:43,686 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:44,686 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:44,811 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:45,263 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:45,686 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:46,687 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:47,263 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:47,687 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:48,687 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:49,264 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:49,687 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:50,683 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:50,687 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:51,687 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:52,687 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:53,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:54,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:51:54,682 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:51:54,725 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:55,266 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:55,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:55,742 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:56,266 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/config.yaml +2025-01-01 13:51:56,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:57,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:58,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:59,267 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:59,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:00,689 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:01,268 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:01,469 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:01,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:02,690 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:03,268 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:03,690 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:04,691 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:05,691 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:06,683 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:06,692 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:07,692 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:08,692 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:09,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:52:09,685 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:52:09,725 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:10,692 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:11,692 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:11,838 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:12,692 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:13,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:14,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:15,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:16,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:17,686 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:17,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:18,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:19,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:20,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:21,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:22,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:22,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:23,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:24,354 DEBUG SystemMonitor:1808069 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 13:52:24,354 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:52:24,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:52:24,682 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:52:24,725 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:25,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:26,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:27,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:27,777 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:28,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:29,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:30,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:31,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:32,695 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:33,276 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:33,497 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:33,695 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:34,695 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:35,276 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:35,695 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:36,695 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:37,316 DEBUG SenderThread:1808069 [sender.py:send():379] send: files +2025-01-01 13:52:37,317 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_294304c5db5cd37da633.mp4 with policy now +2025-01-01 13:52:37,317 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:52:37,318 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:52:37,318 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:52:37,318 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:52:37,618 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/g0ajxrme-media/videos/eval/0_eval_0_294304c5db5cd37da633.mp4 +2025-01-01 13:52:37,695 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:38,277 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:52:38,277 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/0_eval_0_294304c5db5cd37da633.mp4 +2025-01-01 13:52:38,278 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos +2025-01-01 13:52:38,278 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval +2025-01-01 13:52:38,278 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media +2025-01-01 13:52:38,616 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/rgsdswma-media/videos/eval/0_eval_0_294304c5db5cd37da633.mp4 +2025-01-01 13:52:38,696 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:39,278 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:39,329 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:39,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:52:39,683 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:52:39,725 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:40,695 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:41,696 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:42,696 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:43,279 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:43,697 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:44,687 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:44,696 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:45,697 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:46,697 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:47,280 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:47,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:48,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:49,281 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:52:49,539 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/5meirgg6-events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:52:49,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:50,687 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:50,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:51,282 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:51,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:52,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:53,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:54,355 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:52:54,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:52:54,683 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:52:54,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:55,283 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:55,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:55,807 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:56,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:57,284 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:57,699 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:58,699 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:59,285 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:59,699 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:00,699 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:01,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:01,699 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:02,699 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:03,700 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:04,701 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:05,701 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:06,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:06,701 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:07,703 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:08,703 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:09,683 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:53:09,683 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:53:09,725 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:10,704 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:11,288 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:11,704 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:12,593 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:12,704 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:13,289 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:13,704 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:14,704 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:15,706 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:16,705 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:17,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:17,705 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:18,706 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:19,706 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:20,706 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:21,706 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:22,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:22,706 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:23,706 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:24,355 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:53:24,683 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:53:24,683 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:53:24,725 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:25,707 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:26,707 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:27,707 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:27,770 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:28,707 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:29,707 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:30,707 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:31,707 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:32,707 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:32,811 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:33,125 DEBUG SenderThread:1808069 [sender.py:send():379] send: files +2025-01-01 13:53:33,125 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_9e0e7530b69fb1fd581f.mp4 with policy now +2025-01-01 13:53:33,126 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:53:33,127 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:53:33,127 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:53:33,135 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:53:33,311 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:53:33,311 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:33,311 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/1_eval_1_9e0e7530b69fb1fd581f.mp4 +2025-01-01 13:53:33,311 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval +2025-01-01 13:53:33,399 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/wehy69l9-media/videos/eval/1_eval_1_9e0e7530b69fb1fd581f.mp4 +2025-01-01 13:53:33,676 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/bza0zewi-media/videos/eval/1_eval_1_9e0e7530b69fb1fd581f.mp4 +2025-01-01 13:53:33,715 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:34,714 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:35,311 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:35,710 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:36,710 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:37,712 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:38,708 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:38,713 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:39,683 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:53:39,683 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:53:39,725 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:40,715 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:41,721 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:42,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:43,314 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:43,717 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:44,274 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:44,715 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:45,315 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:45,716 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:46,128 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:53:46,128 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:53:46,128 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:53:46,315 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:53:46,315 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:53:46,717 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:47,315 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:47,716 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:48,717 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:49,690 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:49,717 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:50,717 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:51,317 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:51,717 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:52,719 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:53,317 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:53,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:54,356 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:53:54,683 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:53:54,683 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:53:54,717 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:54,752 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:55,318 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:55,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:56,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:57,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:58,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:59,320 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:59,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:00,690 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:00,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:01,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:02,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:03,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:04,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:05,690 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:05,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:06,719 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:07,719 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:08,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:09,683 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:54:09,683 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:54:09,725 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:10,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:10,834 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:11,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:12,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:13,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:14,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:15,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:16,691 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:16,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:17,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:18,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:19,721 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:20,722 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:21,691 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:21,722 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:22,722 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:23,722 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:24,356 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:54:24,684 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:54:24,684 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:54:24,725 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:25,722 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:26,722 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:26,760 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:27,722 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:28,723 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:29,745 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:30,085 DEBUG SenderThread:1808069 [sender.py:send():379] send: files +2025-01-01 13:54:30,085 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a1afdd9f2c1c0e92713f.mp4 with policy now +2025-01-01 13:54:30,088 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:54:30,088 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:54:30,089 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:54:30,095 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:54:30,330 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:54:30,330 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/2_eval_2_a1afdd9f2c1c0e92713f.mp4 +2025-01-01 13:54:30,330 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval +2025-01-01 13:54:30,367 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/kzsylqqp-media/videos/eval/2_eval_2_a1afdd9f2c1c0e92713f.mp4 +2025-01-01 13:54:30,642 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/m7vbr8rt-media/videos/eval/2_eval_2_a1afdd9f2c1c0e92713f.mp4 +2025-01-01 13:54:30,734 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:31,330 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:54:31,730 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:32,152 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:32,728 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:33,728 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:34,733 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:35,731 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:36,728 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:37,460 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:37,728 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:38,732 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:39,333 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:54:39,684 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:54:39,685 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:54:39,735 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:40,736 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:41,333 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:54:41,728 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:42,731 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:43,334 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:54:43,405 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:43,733 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:43,975 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:54:43,975 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:54:43,976 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:54:44,335 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:54:44,335 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:54:44,591 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/hkgep0ui-events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:54:44,735 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:45,335 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:54:45,740 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:46,739 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:47,736 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:48,696 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:48,736 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:49,735 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:50,738 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:51,337 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:54:51,739 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:52,736 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:53,737 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:53,878 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:54,357 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:54:54,684 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:54:54,684 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:54:54,737 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:55,338 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:54:55,737 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:56,737 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:57,737 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:58,737 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:59,696 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:59,737 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:00,738 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:01,738 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:02,738 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:03,738 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:04,696 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:04,738 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:05,738 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:06,739 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:07,739 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:08,739 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:09,684 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:55:09,684 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:55:09,740 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:09,788 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:10,742 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:11,740 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:12,740 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:13,741 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:14,741 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:15,696 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:15,741 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:16,741 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:17,742 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:18,743 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:19,741 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:20,741 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:21,696 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:21,742 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:22,742 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:23,742 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:24,357 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:55:24,684 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:55:24,685 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:55:24,742 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:25,349 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:25,742 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:26,742 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:26,786 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:27,746 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:28,696 DEBUG SenderThread:1808069 [sender.py:send():379] send: files +2025-01-01 13:55:28,696 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_48a28150d9236694be84.mp4 with policy now +2025-01-01 13:55:28,699 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:55:28,699 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:55:28,699 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:55:28,703 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:55:28,747 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:29,024 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/oki45pjc-media/videos/eval/3_eval_3_48a28150d9236694be84.mp4 +2025-01-01 13:55:29,350 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:55:29,350 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:29,350 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/3_eval_3_48a28150d9236694be84.mp4 +2025-01-01 13:55:29,350 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval +2025-01-01 13:55:29,644 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/velh0gc1-media/videos/eval/3_eval_3_48a28150d9236694be84.mp4 +2025-01-01 13:55:29,752 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:30,753 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:31,351 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:31,746 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:32,700 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:32,754 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:33,750 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:34,760 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:35,763 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:36,759 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:37,353 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:37,759 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:38,197 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:38,758 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:39,353 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:39,686 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:55:39,687 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:55:39,761 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:40,760 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:41,354 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:41,767 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:42,769 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:43,772 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:44,002 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:44,785 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:45,044 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:55:45,044 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:55:45,045 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:55:45,355 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:55:45,356 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:55:45,601 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/1iis42j0-events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:55:45,770 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:46,781 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:47,356 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:47,774 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:48,774 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:49,045 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:49,793 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:50,793 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:51,357 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:51,776 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:52,776 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:53,358 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:53,777 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:54,358 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:55:54,358 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:54,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:55:54,688 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:55:54,777 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:55,359 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:55,776 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:56,776 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:57,360 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:57,776 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:58,777 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:59,387 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:59,776 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:00,776 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:01,776 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:02,777 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:03,777 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:04,705 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:04,777 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:05,363 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:05,777 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:06,777 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:07,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:08,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:09,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:56:09,685 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:56:09,761 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:09,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:10,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:11,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:12,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:13,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:14,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:15,711 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:15,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:16,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:17,779 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:18,779 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:19,779 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:20,711 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:20,779 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:21,779 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:22,780 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:23,369 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:23,780 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:24,405 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:56:24,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:56:24,685 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:56:24,782 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:25,776 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:25,782 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:26,782 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:27,782 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:28,782 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:29,782 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:30,787 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:30,811 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:31,304 DEBUG SenderThread:1808069 [sender.py:send():379] send: files +2025-01-01 13:56:31,304 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ae99ac981fb1172cf28a.mp4 with policy now +2025-01-01 13:56:31,309 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:56:31,309 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:56:31,309 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:31,309 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:31,372 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:56:31,373 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:31,373 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/4_eval_4_ae99ac981fb1172cf28a.mp4 +2025-01-01 13:56:31,373 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval +2025-01-01 13:56:31,585 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/10vo79tn-media/videos/eval/4_eval_4_ae99ac981fb1172cf28a.mp4 +2025-01-01 13:56:31,783 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:31,861 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/sdxm1gnv-media/videos/eval/4_eval_4_ae99ac981fb1172cf28a.mp4 +2025-01-01 13:56:32,804 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:33,373 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:33,800 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:34,793 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:35,796 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:36,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:36,796 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:37,806 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:38,809 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:39,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:56:39,694 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:56:39,816 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:40,820 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:41,376 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:41,813 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:42,009 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:42,816 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:43,819 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:44,826 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:45,063 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 13:56:45,063 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 13:56:45,230 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:56:45,377 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:56:45,377 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:45,377 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 13:56:45,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media +2025-01-01 13:56:45,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table/eval +2025-01-01 13:56:45,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table +2025-01-01 13:56:45,526 INFO wandb-upload_0:1808069 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgll3s0ni +2025-01-01 13:56:45,594 INFO wandb-upload_1:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/gid94xsv-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 13:56:45,629 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/xrvogc08-events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:56:45,700 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 13:56:45,713 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:56:45,823 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:45,984 INFO SenderThread:1808069 [sender.py:send_request_log_artifact():1525] logged artifact run-niwmozqn-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDc4MzM2Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI1NTgxMg==', 'latestArtifact': None}} +2025-01-01 13:56:45,985 DEBUG SenderThread:1808069 [sender.py:send():379] send: files +2025-01-01 13:56:45,985 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-01 13:56:45,985 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:56:45,985 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,986 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,986 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:56:45,986 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,990 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,990 DEBUG SenderThread:1808069 [sender.py:send():379] send: telemetry +2025-01-01 13:56:45,990 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,990 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,990 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,991 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,991 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,991 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,991 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,992 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,992 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,992 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,992 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,992 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,993 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,993 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,993 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,993 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,994 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,994 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,994 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,995 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,995 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,996 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,997 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 13:56:46,240 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/aaxshj2u-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 13:56:46,327 INFO wandb-upload_2:1808069 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppqpe0s0d +2025-01-01 13:56:46,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:56:46,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:56:46,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table/eval/total_success_rate_ci_6_246a86f1a070621a1b8e.table.json +2025-01-01 13:56:46,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table/eval +2025-01-01 13:56:46,707 INFO SenderThread:1808069 [sender.py:send_request_log_artifact():1525] logged artifact run-niwmozqn-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDc4MzM2OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI1NTgxNA==', 'latestArtifact': None}} +2025-01-01 13:56:46,708 DEBUG SenderThread:1808069 [sender.py:send():379] send: files +2025-01-01 13:56:46,708 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_246a86f1a070621a1b8e.table.json with policy now +2025-01-01 13:56:46,710 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:56:46,710 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:46,710 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:46,711 DEBUG SenderThread:1808069 [sender.py:send():379] send: exit +2025-01-01 13:56:46,711 INFO SenderThread:1808069 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 13:56:46,711 INFO SenderThread:1808069 [sender.py:send_exit():588] handling runtime: 321 +2025-01-01 13:56:46,714 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:46,714 INFO SenderThread:1808069 [sender.py:send_exit():594] send defer +2025-01-01 13:56:46,715 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:46,715 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 13:56:46,715 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:46,715 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 13:56:46,715 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 1 +2025-01-01 13:56:46,715 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:46,715 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 13:56:46,715 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:46,715 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 13:56:46,715 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 2 +2025-01-01 13:56:46,715 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:46,715 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 13:56:46,716 INFO HandlerThread:1808069 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 13:56:46,716 DEBUG SystemMonitor:1808069 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 13:56:46,716 DEBUG SystemMonitor:1808069 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 13:56:46,716 INFO HandlerThread:1808069 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 13:56:46,717 INFO HandlerThread:1808069 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 13:56:46,779 INFO HandlerThread:1808069 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 13:56:46,779 INFO HandlerThread:1808069 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 13:56:46,779 INFO HandlerThread:1808069 [interfaces.py:finish():200] Joined network monitor +2025-01-01 13:56:46,780 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:46,780 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 13:56:46,780 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 3 +2025-01-01 13:56:46,780 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:56:46,780 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:46,780 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 13:56:46,780 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:46,780 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 13:56:46,780 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 4 +2025-01-01 13:56:46,781 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:46,781 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 13:56:46,912 INFO wandb-upload_1:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/ch43452c-media/table/eval/total_success_rate_ci_6_246a86f1a070621a1b8e.table.json +2025-01-01 13:56:47,171 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/eev4frtl-media/table/eval/total_success_rate_ci_6_246a86f1a070621a1b8e.table.json +2025-01-01 13:56:47,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:56:47,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:48,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/config.yaml +2025-01-01 13:56:49,379 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:53,121 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:56:53,121 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:53,121 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 13:56:53,121 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:53,122 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:53,122 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 13:56:53,122 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 5 +2025-01-01 13:56:53,122 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:56:53,123 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:56:53,123 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:53,123 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:53,123 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 13:56:53,124 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:53,124 DEBUG SenderThread:1808069 [sender.py:send():379] send: summary +2025-01-01 13:56:53,124 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:53,124 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:53,124 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 13:56:53,124 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 6 +2025-01-01 13:56:53,125 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:53,125 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 13:56:53,125 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:53,125 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 13:56:53,125 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 7 +2025-01-01 13:56:53,125 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:53,125 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:53,125 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 13:56:53,125 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:53,125 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 13:56:53,230 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:56:53,381 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:56:55,369 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 8 +2025-01-01 13:56:55,369 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:56:55,369 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:55,369 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 13:56:55,369 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:55,370 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 13:56:55,370 INFO SenderThread:1808069 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 13:56:55,374 INFO SenderThread:1808069 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 13:56:55,376 INFO SenderThread:1808069 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 13:56:55,381 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:55,386 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 9 +2025-01-01 13:56:55,386 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:55,386 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 13:56:55,386 DEBUG SenderThread:1808069 [sender.py:send():379] send: artifact +2025-01-01 13:56:55,551 INFO SenderThread:1808069 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIxODYyMQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 13:56:55,551 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:55,551 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 13:56:55,551 INFO SenderThread:1808069 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 13:56:56,232 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:56:56,382 INFO SenderThread:1808069 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files +2025-01-01 13:56:56,382 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/config.yaml config.yaml +2025-01-01 13:56:56,383 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json wandb-summary.json +2025-01-01 13:56:56,383 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:56:56,383 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/requirements.txt requirements.txt +2025-01-01 13:56:56,383 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-metadata.json wandb-metadata.json +2025-01-01 13:56:56,383 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log output.log +2025-01-01 13:56:56,383 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/diff.patch diff.patch +2025-01-01 13:56:56,386 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 13:56:56,386 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/3_eval_3_48a28150d9236694be84.mp4 media/videos/eval/3_eval_3_48a28150d9236694be84.mp4 +2025-01-01 13:56:56,386 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/4_eval_4_ae99ac981fb1172cf28a.mp4 media/videos/eval/4_eval_4_ae99ac981fb1172cf28a.mp4 +2025-01-01 13:56:56,388 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/2_eval_2_a1afdd9f2c1c0e92713f.mp4 media/videos/eval/2_eval_2_a1afdd9f2c1c0e92713f.mp4 +2025-01-01 13:56:56,388 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/1_eval_1_9e0e7530b69fb1fd581f.mp4 media/videos/eval/1_eval_1_9e0e7530b69fb1fd581f.mp4 +2025-01-01 13:56:56,388 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/0_eval_0_294304c5db5cd37da633.mp4 media/videos/eval/0_eval_0_294304c5db5cd37da633.mp4 +2025-01-01 13:56:56,391 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table/eval/total_success_rate_ci_6_246a86f1a070621a1b8e.table.json media/table/eval/total_success_rate_ci_6_246a86f1a070621a1b8e.table.json +2025-01-01 13:56:56,391 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 13:56:56,391 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 10 +2025-01-01 13:56:56,391 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:56:56,391 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:56,391 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 13:56:56,392 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:56,392 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 13:56:56,392 INFO SenderThread:1808069 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 13:56:56,608 INFO wandb-upload_3:1808069 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/requirements.txt +2025-01-01 13:56:56,609 INFO wandb-upload_4:1808069 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:56,610 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:56:56,610 INFO wandb-upload_1:1808069 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/config.yaml +2025-01-01 13:56:56,629 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/6ye2f0b5-events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:56:56,829 INFO Thread-11 (_thread_body):1808069 [sender.py:transition_state():614] send defer: 11 +2025-01-01 13:56:56,829 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:56,829 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 13:56:56,830 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:56,830 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 13:56:56,830 INFO SenderThread:1808069 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 13:56:56,831 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 12 +2025-01-01 13:56:56,831 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:56,831 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 13:56:56,831 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:56,831 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 13:56:56,831 INFO SenderThread:1808069 [file_stream.py:finish():601] file stream finish called +2025-01-01 13:56:57,070 INFO SenderThread:1808069 [file_stream.py:finish():605] file stream finish is done +2025-01-01 13:56:57,070 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 13 +2025-01-01 13:56:57,070 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:57,070 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 13:56:57,070 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:57,070 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 13:56:57,070 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 14 +2025-01-01 13:56:57,070 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:57,070 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 13:56:57,071 DEBUG SenderThread:1808069 [sender.py:send():379] send: final +2025-01-01 13:56:57,071 DEBUG SenderThread:1808069 [sender.py:send():379] send: footer +2025-01-01 13:56:57,071 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:57,071 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 13:56:57,071 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:56:57,071 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:56:57,072 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 13:56:57,072 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:56:57,072 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: server_info +2025-01-01 13:56:57,074 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 13:56:57,074 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 13:56:57,075 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:57,136 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:56:57,136 INFO MainThread:1808069 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 13:56:57,137 INFO MainThread:1808069 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 13:56:57,137 INFO MainThread:1808069 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 13:56:57,137 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 13:56:57,137 INFO HandlerThread:1808069 [handler.py:finish():882] shutting down handler +2025-01-01 13:56:58,074 INFO WriterThread:1808069 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/run-niwmozqn.wandb +2025-01-01 13:56:58,136 INFO SenderThread:1808069 [sender.py:finish():1615] shutting down sender +2025-01-01 13:56:58,136 INFO SenderThread:1808069 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 13:56:58,136 INFO SenderThread:1808069 [file_pusher.py:join():175] waiting for file pusher diff --git a/niwmozqn/wandb/debug.log b/niwmozqn/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..224d1ba36a0c950c55406476bb5426aa32a5eef8 --- /dev/null +++ b/niwmozqn/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_setup.py:_flush():76] Configure stats pid to 1807895 +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/settings +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/logs/debug.log +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/logs/debug-internal.log +2025-01-01 13:51:24,027 INFO MainThread:1807895 [wandb_init.py:init():569] calling init triggers +2025-01-01 13:51:24,027 INFO MainThread:1807895 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn'} +2025-01-01 13:51:24,027 INFO MainThread:1807895 [wandb_init.py:init():619] starting backend +2025-01-01 13:51:24,027 INFO MainThread:1807895 [wandb_init.py:init():623] setting up manager +2025-01-01 13:51:24,032 INFO MainThread:1807895 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 13:51:24,033 INFO MainThread:1807895 [wandb_init.py:init():631] backend started and connected +2025-01-01 13:51:24,046 INFO MainThread:1807895 [wandb_init.py:init():720] updated telemetry +2025-01-01 13:51:24,055 INFO MainThread:1807895 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 13:51:24,271 INFO MainThread:1807895 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 13:51:24,341 INFO MainThread:1807895 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 13:51:24,341 INFO MainThread:1807895 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 13:51:24,682 INFO MainThread:1807895 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 13:51:24,682 INFO MainThread:1807895 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 13:51:24,682 INFO MainThread:1807895 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 13:51:24,682 INFO MainThread:1807895 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 13:51:24,686 INFO MainThread:1807895 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 13:51:24,686 INFO MainThread:1807895 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/tb, True +2025-01-01 13:51:30,015 INFO MainThread:1807895 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-01 13:56:58,138 WARNING MsgRouterThr:1807895 [router.py:message_loop():77] message_loop has been closed diff --git a/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/code/FISH/eval_robot.py b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/config.yaml b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64f6a28a73de1f084cd2bffb3342850be0ad85e0 --- /dev/null +++ b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1745_0' +wandb_notes: + desc: null + value: 1745_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735757484 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/diff.patch b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..7f9cbfaf74a18317e7403f01a665bc5e3d98239c --- /dev/null +++ b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/diff.patch @@ -0,0 +1,106 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0f6a9ec..57a851f 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1007_0' ++# wandb_run_id: '1002_0' + ++# all books, 6/20 demos per book ++# RGBD+mask+act history ++# wandb_run_id: '1007_0' ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1006_0' ++ ++# RGBD+mask+contact(contact+DTC)+act history ++wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 21e335c..71a5d1c 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -155,7 +155,12 @@ run_id_list = [ + # '1022_0', + # '1023_0', + # '1007_0', +- '1007_0' ++ # '1007_0' ++] ++ ++run_id_list = [ ++ # '1006_0', ++ '1745_0' + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 4179186..50493fe 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -478,8 +478,8 @@ class Workspace: + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", +- "config.grasped_obj_name": grasped_obj_name, +- "config.left_book_slot": left_book_slot, ++ # "config.grasped_obj_name": grasped_obj_name, ++ # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() +@@ -490,7 +490,8 @@ class Workspace: + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] +- total_num_episodes += filtered_run.summary_metrics['episode'] ++ # total_num_episodes += filtered_run.summary_metrics['episode'] ++ total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes +@@ -513,7 +514,6 @@ class Workspace: + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + +- + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) +diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py +index 3302c31..1946eca 100644 +--- a/gym-envs/gym_envs/envs/franka_env.py ++++ b/gym-envs/gym_envs/envs/franka_env.py +@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env): + + def mask_obs_callback(self, msg): + msg = self.possibly_resize_mask_obs(msg) +- rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") ++ # rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}") + self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg) + + def contact_obs_callback(self, msg): +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index e3031af..6f97e26 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,11 +330,11 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.01 +- self.move_to_rot_tolerance = 2.0 ++ self.move_to_trans_tolerance = 0.005 ++ self.move_to_rot_tolerance = 1.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) +- self.move_to_target_timeout = 15.0 ++ self.move_to_target_timeout = 5.0 + self.move_to_target_max_rotation = 25.0 + # only can use one of these at a time + # assert use_roll + use_pitch + use_yaw <= 1, "Can only use one of roll, pitch, yaw" diff --git a/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table/eval/total_success_rate_ci_6_246a86f1a070621a1b8e.table.json b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table/eval/total_success_rate_ci_6_246a86f1a070621a1b8e.table.json new file mode 100644 index 0000000000000000000000000000000000000000..e62bf59dfe43953634623d2ca77c1e940a0e0414 --- /dev/null +++ b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table/eval/total_success_rate_ci_6_246a86f1a070621a1b8e.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[29, 60, 0.48333333333333334, 0.33502373102626615, 0.6326915770503736, 0.36239646377931367, 0.615297824121304, 0.3812942356108612, 0.5864350572055562, 0.39682678135312754, 0.5718618966495318]]} \ No newline at end of file diff --git a/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/0_eval_0_294304c5db5cd37da633.mp4 b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/0_eval_0_294304c5db5cd37da633.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c695b2a3050ba2f62e7796e1526f1a7d0969d29e --- /dev/null +++ b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/0_eval_0_294304c5db5cd37da633.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294304c5db5cd37da633dec9ddbed006a3ffe4d2a3c0f2850acd77dfc03696cd +size 1464499 diff --git a/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/1_eval_1_9e0e7530b69fb1fd581f.mp4 b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/1_eval_1_9e0e7530b69fb1fd581f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..978272677e77d8fc2739615d8c28885b0719c18a --- /dev/null +++ b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/1_eval_1_9e0e7530b69fb1fd581f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e0e7530b69fb1fd581f0857a472524c725b947703abc37411eaf1190e11b9ea +size 1455848 diff --git a/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/2_eval_2_a1afdd9f2c1c0e92713f.mp4 b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/2_eval_2_a1afdd9f2c1c0e92713f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3ccda36def106f6f765aeca58c1756dab5890a62 --- /dev/null +++ b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/2_eval_2_a1afdd9f2c1c0e92713f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1afdd9f2c1c0e92713f213a35100539fed551b2404536f444a52c463d048e22 +size 1449330 diff --git a/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/3_eval_3_48a28150d9236694be84.mp4 b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/3_eval_3_48a28150d9236694be84.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..73647a39298f364bce83b8b9ddd429d1a300b828 --- /dev/null +++ b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/3_eval_3_48a28150d9236694be84.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48a28150d9236694be84fb1c4293ccbe9972c28906ca6e70f627090a7e12c862 +size 1454516 diff --git a/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/4_eval_4_ae99ac981fb1172cf28a.mp4 b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/4_eval_4_ae99ac981fb1172cf28a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4fa7d88f349e1be2e4eb85be1b1bb9b204ad5dda --- /dev/null +++ b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/4_eval_4_ae99ac981fb1172cf28a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae99ac981fb1172cf28a49dfa5fa49ad6ad047d62657a8efbb3c9744f75674a6 +size 1513871 diff --git a/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c84a0394cbeef35cac3ca8a5167ba78b913cac32 --- /dev/null +++ b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log @@ -0,0 +1,280 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735757490.018957]: resetting environment +[INFO] [1735757490.028114]: cleared current plan +[INFO] [1735757490.029622]: moving to home +Move to target timeout: 5.000730514526367. Trans error: 0.005146494829883841. Rot error: 0.8908692204286511 +[INFO] [1735757496.033135]: reached home +[INFO] [1735757496.034475]: reset action history +[INFO] [1735757498.797714]: environment reset +Starting episode 0 +[INFO] [1735757498.802195]: resetting environment +[INFO] [1735757498.806410]: cleared current plan +[INFO] [1735757498.807253]: moving to home +Move to target timeout: 5.000765323638916. Trans error: 0.005143983777843018. Rot error: 0.8903042900572782 +[INFO] [1735757504.810914]: reached home +[INFO] [1735757504.812829]: reset action history +[INFO] [1735757507.523381]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735757512.751828]: resetting environment +[INFO] [1735757512.756183]: cleared current plan +[INFO] [1735757512.757055]: moving to home +Move to target timeout: 5.000649452209473. Trans error: 0.005143385533319378. Rot error: 0.8895238905527842 +[INFO] [1735757518.759971]: reached home +[INFO] [1735757518.761377]: reset action history +[INFO] [1735757521.469339]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/episode_rosbags/episode_0_2025-01-01-13-52-01.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1807895.log +started roslaunch server http://158.130.50.37:44209/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1808176] +process[print_text-2]: started with pid [1808200] +process[rosbag_record-3]: started with pid [1808201] +started bagging! +For topic gripper_width: timestamp difference is 102221259 for nearest: 1735757550352800855 - target: 1735757550250579596 at idx 249 +For topic gripper_width: timestamp difference is 102872784 for nearest: 1735757551152804072 - target: 1735757551049931288 at idx 249 +For topic gripper_width: timestamp difference is 103882136 for nearest: 1735757552452766242 - target: 1735757552348884106 at idx 249 +[INFO] [1735757556.366864]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735757557.119224]: Stored episode 1. +[INFO] [1735757557.120195]: Saving video... +[INFO] [1735757557.307107]: Video saved! +Starting episode 1 +[INFO] [1735757557.319354]: resetting environment +[INFO] [1735757557.327728]: cleared current plan +[INFO] [1735757557.328442]: moving to home +[INFO] [1735757562.132064]: reached home +[INFO] [1735757562.132847]: reset action history +[INFO] [1735757564.842221]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735757568.547537]: resetting environment +[INFO] [1735757568.550765]: cleared current plan +[INFO] [1735757568.550932]: moving to home +Move to target timeout: 5.000713348388672. Trans error: 0.005142904659665349. Rot error: 0.8986219367675568 +[INFO] [1735757574.553213]: reached home +[INFO] [1735757574.554188]: reset action history +[INFO] [1735757577.262032]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/episode_rosbags/episode_1_2025-01-01-13-52-57.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1807895.log +started roslaunch server http://158.130.50.37:38229/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1808326] +process[print_text-5]: started with pid [1808327] +process[rosbag_record-6]: started with pid [1808351] +started bagging! +For topic gripper_width: timestamp difference is 101707905 for nearest: 1735757590252805203 - target: 1735757590151097298 at idx 178 +For topic gripper_width: timestamp difference is 104371156 for nearest: 1735757591552808608 - target: 1735757591448437452 at idx 193 +[INFO] [1735757612.015734]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735757612.811453]: Stored episode 2. +[INFO] [1735757612.811795]: Saving video... +[INFO] [1735757613.101255]: Video saved! +Starting episode 2 +[INFO] [1735757613.128633]: resetting environment +[INFO] [1735757613.144247]: cleared current plan +[INFO] [1735757613.144745]: moving to home +Move to target timeout: 5.000169515609741. Trans error: 0.005075834802610572. Rot error: 0.859418029678707 +[INFO] [1735757621.549172]: reached home +[INFO] [1735757621.549835]: reset action history +[INFO] [1735757624.274128]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735757625.408505]: resetting environment +[INFO] [1735757625.420244]: cleared current plan +[INFO] [1735757625.420501]: moving to home +Move to target timeout: 5.001283884048462. Trans error: 0.005062532760707794. Rot error: 0.8504745951546699 +[INFO] [1735757631.423022]: reached home +[INFO] [1735757631.423494]: reset action history +[INFO] [1735757634.136023]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/episode_rosbags/episode_2_2025-01-01-13-53-54.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1807895.log +started roslaunch server http://158.130.50.37:43863/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1808470] +process[print_text-8]: started with pid [1808471] +process[rosbag_record-9]: started with pid [1808495] +started bagging! +For topic gripper_width: timestamp difference is 105531710 for nearest: 1735757636952833193 - target: 1735757636847301483 at idx 35 +[INFO] [1735757668.876706]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit[print_text-8] killing on exit +[INFO] [1735757669.690636]: Stored episode 3. +[INFO] [1735757669.699371]: Saving video... +[INFO] [1735757670.040025]: Video saved! +Starting episode 3 +[INFO] [1735757670.097485]: resetting environment +[INFO] [1735757670.141683]: cleared current plan +[INFO] [1735757670.151101]: moving to home +Move to target timeout: 5.000189781188965. Trans error: 0.005104743336018951. Rot error: 0.8597427962430129 +[INFO] [1735757678.462258]: reached home +[INFO] [1735757678.469396]: reset action history +[INFO] [1735757681.357172]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735757683.834582]: resetting environment +[INFO] [1735757683.862563]: cleared current plan +[INFO] [1735757683.865640]: moving to home +Move to target timeout: 5.004344463348389. Trans error: 0.005083024686691201. Rot error: 0.8499946675519111 +[INFO] [1735757689.875144]: reached home +[INFO] [1735757689.875883]: reset action history +[INFO] [1735757692.603963]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/episode_rosbags/episode_3_2025-01-01-13-54-52.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1807895.log +started roslaunch server http://158.130.50.37:38951/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1808612] +process[print_text-11]: started with pid [1808613] +process[rosbag_record-12]: started with pid [1808637] +started bagging! +For topic gripper_width: timestamp difference is -102988803 for nearest: 1735757722786137021 - target: 1735757722889125824 at idx 249 +[INFO] [1735757727.394729]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1735757728.185619]: Stored episode 4. +[INFO] [1735757728.187889]: Saving video... +[INFO] [1735757728.638780]: Video saved! +Starting episode 4 +[INFO] [1735757728.704307]: resetting environment +[INFO] [1735757728.736514]: cleared current plan +[INFO] [1735757728.736778]: moving to home +Move to target timeout: 5.002671718597412. Trans error: 0.005115635017185513. Rot error: 0.8745892399118254 +[INFO] [1735757737.153133]: reached home +[INFO] [1735757737.164440]: reset action history +[INFO] [1735757739.958937]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735757744.704286]: resetting environment +[INFO] [1735757744.739142]: cleared current plan +[INFO] [1735757744.743078]: moving to home +Move to target timeout: 5.001695394515991. Trans error: 0.005103623547113961. Rot error: 0.8652898406054975 +[INFO] [1735757750.749213]: reached home +[INFO] [1735757750.753599]: reset action history +[INFO] [1735757753.535632]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/episode_rosbags/episode_4_2025-01-01-13-55-53.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1807895.log +started roslaunch server http://158.130.50.37:33673/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1808777] +process[print_text-14]: started with pid [1808778] +process[rosbag_record-15]: started with pid [1808779] +started bagging! +For topic gripper_width: timestamp difference is -196079484 for nearest: 1735757762386162290 - target: 1735757762582241774 at idx 133 +For topic gripper_width: timestamp difference is -211308334 for nearest: 1735757780886135485 - target: 1735757781097443819 at idx 249 +[INFO] [1735757790.002289]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit[print_text-14] killing on exit +[INFO] [1735757790.810089]: Stored episode 5. +[INFO] [1735757790.812431]: Saving video... +[INFO] [1735757791.245832]: Video saved! +[INFO] [1735757791.310931]: resetting environment +[INFO] [1735757791.342343]: cleared current plan +[INFO] [1735757791.342540]: moving to home +[WARN] [1735757791.340262]: Plan exhausted +Move to target timeout: 5.097748517990112. Trans error: 0.005107580682179898. Rot error: 0.8666707556157643 +[INFO] [1735757799.958502]: reached home +[INFO] [1735757799.970370]: reset action history +[INFO] [1735757802.759904]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/requirements.txt b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-metadata.json b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d60dcbf85c433df4e2a90b651722f1b5f8a0f5eb --- /dev/null +++ b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T18:51:24.363514", + "startedAt": "2025-01-01T18:51:24.020448", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.5651249999996, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.922, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 811.3620529174805 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..242dec6cfbe0792662a0507134dcdb4306a3362c --- /dev/null +++ b/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "294304c5db5cd37da633dec9ddbed006a3ffe4d2a3c0f2850acd77dfc03696cd", "size": 1464499, "path": "media/videos/eval/0_eval_0_294304c5db5cd37da633.mp4"}, "global_step": 665, "_timestamp": 1735757805.609578, "_runtime": 321.5755069255829, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "9e0e7530b69fb1fd581f0857a472524c725b947703abc37411eaf1190e11b9ea", "size": 1455848, "path": "media/videos/eval/1_eval_1_9e0e7530b69fb1fd581f.mp4"}, "eval/num_success": 5.0, "episode": 60.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "a1afdd9f2c1c0e92713f213a35100539fed551b2404536f444a52c463d048e22", "size": 1449330, "path": "media/videos/eval/2_eval_2_a1afdd9f2c1c0e92713f.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "48a28150d9236694be84fb1c4293ccbe9972c28906ca6e70f627090a7e12c862", "size": 1454516, "path": "media/videos/eval/3_eval_3_48a28150d9236694be84.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "ae99ac981fb1172cf28a49dfa5fa49ad6ad047d62657a8efbb3c9744f75674a6", "size": 1513871, "path": "media/videos/eval/4_eval_4_ae99ac981fb1172cf28a.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://xczwvcc4f5x6awl2v8wmkqoyitq94js4ih85rfk55kbb4un0xxiwsixvp2bul99vqo58iykygxfvqcf340m34ecibn6y4mobg57h90m9kgxoaigq6p786t2yjgntoqt2/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://70dmbrjlkg12qjrh94zkeb2h1pnklprchl4hy0cll0xcvwlscxoml89wlbffivdprs7xb9jhrz2picuohaajie1nt3czu6coy65kmudqg0njkxtthrf0uxdw6i1v5n6x:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 29, "total_num_episodes": 60, "total_success_rate": 0.48333333333333334, "total_success_rate_lb_0.01": 0.33502373102626615, "total_success_rate_ub_0.01": 0.6326915770503736, "total_success_rate_lb_0.025": 0.36239646377931367, "total_success_rate_ub_0.025": 0.615297824121304, "total_success_rate_lb_0.05": 0.3812942356108612, "total_success_rate_ub_0.05": 0.5864350572055562, "total_success_rate_lb_0.1": 0.39682678135312754, "total_success_rate_ub_0.1": 0.5718618966495318, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "246a86f1a070621a1b8e8d52ab0f9388e608c92fcdcdaf3ffb63e097df4d52e6", "size": 523, "artifact_path": "wandb-client-artifact://fwnwnsod09gvqs5c6zd3cq9wubjue7qwbv0g4oa5vudsfehoyyyfbznlmbd7ym9rlyjhwt2p7zjfyhrbdtzlavs2mqi8mpsccl9sk8outpb5b7vj021y3g5ynwgnnrju/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://2i86prsgoep8kfpypct8llabomq5viwfvzbqm44lfdtscr80eh8yqbcntizbbh3hx3g6js20j1zlhqkb2rkj2tsdwwh6y4xse6xs63njjgsw9uowkv543b47l03rel7a:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_246a86f1a070621a1b8e.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 321}, "eval/total_success_rate": 0.4833333194255829, "eval/total_success_rate_lb_0.01": 0.3350237309932709, "eval/total_success_rate_ub_0.01": 0.6326915621757507, "eval/total_success_rate_lb_0.025": 0.3623964786529541, "eval/total_success_rate_ub_0.025": 0.615297794342041, "eval/total_success_rate_lb_0.05": 0.38129425048828125, "eval/total_success_rate_ub_0.05": 0.586435079574585, "eval/total_success_rate_lb_0.1": 0.39682677388191223, "eval/total_success_rate_ub_0.1": 0.5718619227409363} \ No newline at end of file diff --git a/niwmozqn/wandb/run-20250101_135124-niwmozqn/logs/debug-internal.log b/niwmozqn/wandb/run-20250101_135124-niwmozqn/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..0dc10a42d20d19c7f4e8fe12a4621daca667d0d2 --- /dev/null +++ b/niwmozqn/wandb/run-20250101_135124-niwmozqn/logs/debug-internal.log @@ -0,0 +1,854 @@ +2025-01-01 13:51:24,033 INFO StreamThr :1808069 [internal.py:wandb_internal():85] W&B internal server running at pid: 1808069, started at: 2025-01-01 13:51:24.033568 +2025-01-01 13:51:24,034 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status +2025-01-01 13:51:24,047 INFO WriterThread:1808069 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/run-niwmozqn.wandb +2025-01-01 13:51:24,047 DEBUG SenderThread:1808069 [sender.py:send():379] send: header +2025-01-01 13:51:24,058 DEBUG SenderThread:1808069 [sender.py:send():379] send: run +2025-01-01 13:51:24,257 INFO SenderThread:1808069 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files +2025-01-01 13:51:24,257 INFO SenderThread:1808069 [sender.py:_start_run_threads():1188] run started: niwmozqn with start time 1735757484.034071 +2025-01-01 13:51:24,271 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 13:51:24,271 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: check_version +2025-01-01 13:51:24,345 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 13:51:24,350 DEBUG HandlerThread:1808069 [system_info.py:__init__():26] System info init +2025-01-01 13:51:24,350 DEBUG HandlerThread:1808069 [system_info.py:__init__():41] System info init done +2025-01-01 13:51:24,350 INFO HandlerThread:1808069 [system_monitor.py:start():194] Starting system monitor +2025-01-01 13:51:24,350 INFO SystemMonitor:1808069 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 13:51:24,351 INFO HandlerThread:1808069 [system_monitor.py:probe():214] Collecting system info +2025-01-01 13:51:24,351 INFO SystemMonitor:1808069 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 13:51:24,352 INFO SystemMonitor:1808069 [interfaces.py:start():188] Started disk monitoring +2025-01-01 13:51:24,353 INFO SystemMonitor:1808069 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 13:51:24,353 INFO SystemMonitor:1808069 [interfaces.py:start():188] Started memory monitoring +2025-01-01 13:51:24,354 INFO SystemMonitor:1808069 [interfaces.py:start():188] Started network monitoring +2025-01-01 13:51:24,363 DEBUG HandlerThread:1808069 [system_info.py:probe():152] Probing system +2025-01-01 13:51:24,364 DEBUG HandlerThread:1808069 [system_info.py:_probe_git():137] Probing git +2025-01-01 13:51:24,369 DEBUG HandlerThread:1808069 [system_info.py:_probe_git():145] Probing git done +2025-01-01 13:51:24,369 DEBUG HandlerThread:1808069 [system_info.py:probe():200] Probing system done +2025-01-01 13:51:24,369 DEBUG HandlerThread:1808069 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T18:51:24.363514', 'startedAt': '2025-01-01T18:51:24.020448', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.5651249999996, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.922, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 811.3620529174805}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 13:51:24,369 INFO HandlerThread:1808069 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 13:51:24,369 INFO HandlerThread:1808069 [system_monitor.py:probe():227] Publishing system info +2025-01-01 13:51:24,369 DEBUG HandlerThread:1808069 [system_info.py:_save_code():44] Saving code +2025-01-01 13:51:24,371 DEBUG HandlerThread:1808069 [system_info.py:_save_code():67] Saving code done +2025-01-01 13:51:24,371 DEBUG HandlerThread:1808069 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 13:51:24,432 DEBUG HandlerThread:1808069 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 13:51:24,433 INFO HandlerThread:1808069 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 13:51:24,436 DEBUG SenderThread:1808069 [sender.py:send():379] send: files +2025-01-01 13:51:24,436 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 13:51:24,436 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 13:51:24,437 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 13:51:24,681 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 13:51:24,682 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: python_packages +2025-01-01 13:51:24,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:51:24,682 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:51:24,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:24,686 INFO HandlerThread:1808069 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/tb" + save: true +} +control { + relay_id: "fc69491f6f894c9d849a333e2dc167c9" +} +_info { + stream_id: "niwmozqn" +} + +2025-01-01 13:51:24,687 DEBUG HandlerThread:1808069 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 13:51:24,687 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/ih4gd4sf-wandb-metadata.json +2025-01-01 13:51:24,707 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/t49br4cp-diff.patch +2025-01-01 13:51:24,753 DEBUG SenderThread:1808069 [sender.py:send():379] send: telemetry +2025-01-01 13:51:24,753 DEBUG SenderThread:1808069 [sender.py:send():379] send: tbrecord +2025-01-01 13:51:24,753 DEBUG SenderThread:1808069 [sender.py:send():379] send: files +2025-01-01 13:51:24,753 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 with policy live +2025-01-01 13:51:24,772 INFO wandb-upload_1:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/kpxyti30-code/FISH/eval_robot.py +2025-01-01 13:51:24,955 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/mpasxk2g-events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:51:25,258 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:25,258 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-metadata.json +2025-01-01 13:51:25,259 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/diff.patch +2025-01-01 13:51:25,259 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/requirements.txt +2025-01-01 13:51:25,259 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:51:25,259 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/code/FISH/eval_robot.py +2025-01-01 13:51:25,259 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/code/FISH +2025-01-01 13:51:25,259 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/code +2025-01-01 13:51:25,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:26,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:27,257 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:27,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:28,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:29,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:29,684 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:30,016 DEBUG SenderThread:1808069 [sender.py:send():379] send: config +2025-01-01 13:51:30,684 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:31,259 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:31,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:32,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:33,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:34,683 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:34,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:35,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:36,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:37,260 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:37,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:38,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:39,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:51:39,682 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:51:39,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:39,803 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:40,686 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:41,262 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:41,686 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:42,686 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:43,686 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:44,686 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:44,811 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:45,263 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:45,686 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:46,687 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:47,263 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:47,687 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:48,687 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:49,264 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:49,687 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:50,683 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:50,687 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:51,687 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:52,687 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:53,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:54,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:51:54,682 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:51:54,725 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:55,266 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:55,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:55,742 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:51:56,266 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/config.yaml +2025-01-01 13:51:56,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:57,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:58,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:51:59,267 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:51:59,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:00,689 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:01,268 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:01,469 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:01,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:02,690 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:03,268 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:03,690 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:04,691 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:05,691 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:06,683 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:06,692 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:07,692 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:08,692 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:09,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:52:09,685 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:52:09,725 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:10,692 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:11,692 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:11,838 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:12,692 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:13,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:14,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:15,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:16,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:17,686 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:17,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:18,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:19,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:20,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:21,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:22,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:22,693 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:23,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:24,354 DEBUG SystemMonitor:1808069 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 13:52:24,354 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:52:24,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:52:24,682 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:52:24,725 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:25,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:26,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:27,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:27,777 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:28,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:29,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:30,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:31,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:32,695 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:33,276 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:33,497 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:33,695 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:34,695 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:35,276 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:35,695 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:36,695 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:37,316 DEBUG SenderThread:1808069 [sender.py:send():379] send: files +2025-01-01 13:52:37,317 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_294304c5db5cd37da633.mp4 with policy now +2025-01-01 13:52:37,317 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:52:37,318 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:52:37,318 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:52:37,318 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:52:37,618 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/g0ajxrme-media/videos/eval/0_eval_0_294304c5db5cd37da633.mp4 +2025-01-01 13:52:37,695 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:38,277 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:52:38,277 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/0_eval_0_294304c5db5cd37da633.mp4 +2025-01-01 13:52:38,278 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos +2025-01-01 13:52:38,278 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval +2025-01-01 13:52:38,278 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media +2025-01-01 13:52:38,616 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/rgsdswma-media/videos/eval/0_eval_0_294304c5db5cd37da633.mp4 +2025-01-01 13:52:38,696 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:39,278 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:39,329 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:39,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:52:39,683 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:52:39,725 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:40,695 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:41,696 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:42,696 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:43,279 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:43,697 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:44,687 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:44,696 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:45,697 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:46,697 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:47,280 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:47,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:48,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:49,281 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:52:49,539 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/5meirgg6-events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:52:49,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:50,687 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:50,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:51,282 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:51,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:52,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:53,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:54,355 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:52:54,682 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:52:54,683 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:52:54,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:55,283 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:55,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:55,807 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:52:56,698 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:57,284 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:57,699 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:58,699 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:52:59,285 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:52:59,699 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:00,699 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:01,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:01,699 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:02,699 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:03,700 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:04,701 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:05,701 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:06,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:06,701 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:07,703 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:08,703 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:09,683 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:53:09,683 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:53:09,725 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:10,704 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:11,288 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:11,704 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:12,593 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:12,704 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:13,289 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:13,704 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:14,704 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:15,706 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:16,705 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:17,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:17,705 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:18,706 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:19,706 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:20,706 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:21,706 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:22,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:22,706 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:23,706 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:24,355 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:53:24,683 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:53:24,683 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:53:24,725 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:25,707 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:26,707 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:27,707 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:27,770 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:28,707 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:29,707 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:30,707 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:31,707 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:32,707 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:32,811 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:33,125 DEBUG SenderThread:1808069 [sender.py:send():379] send: files +2025-01-01 13:53:33,125 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_9e0e7530b69fb1fd581f.mp4 with policy now +2025-01-01 13:53:33,126 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:53:33,127 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:53:33,127 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:53:33,135 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:53:33,311 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:53:33,311 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:33,311 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/1_eval_1_9e0e7530b69fb1fd581f.mp4 +2025-01-01 13:53:33,311 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval +2025-01-01 13:53:33,399 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/wehy69l9-media/videos/eval/1_eval_1_9e0e7530b69fb1fd581f.mp4 +2025-01-01 13:53:33,676 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/bza0zewi-media/videos/eval/1_eval_1_9e0e7530b69fb1fd581f.mp4 +2025-01-01 13:53:33,715 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:34,714 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:35,311 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:35,710 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:36,710 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:37,712 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:38,708 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:38,713 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:39,683 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:53:39,683 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:53:39,725 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:40,715 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:41,721 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:42,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:43,314 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:43,717 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:44,274 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:44,715 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:45,315 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:45,716 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:46,128 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:53:46,128 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:53:46,128 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:53:46,315 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:53:46,315 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:53:46,717 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:47,315 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:47,716 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:48,717 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:49,690 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:49,717 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:50,717 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:51,317 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:51,717 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:52,719 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:53,317 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:53,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:54,356 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:53:54,683 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:53:54,683 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:53:54,717 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:54,752 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:53:55,318 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:55,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:56,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:57,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:58,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:53:59,320 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:53:59,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:00,690 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:00,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:01,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:02,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:03,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:04,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:05,690 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:05,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:06,719 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:07,719 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:08,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:09,683 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:54:09,683 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:54:09,725 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:10,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:10,834 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:11,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:12,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:13,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:14,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:15,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:16,691 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:16,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:17,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:18,720 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:19,721 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:20,722 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:21,691 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:21,722 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:22,722 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:23,722 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:24,356 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:54:24,684 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:54:24,684 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:54:24,725 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:25,722 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:26,722 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:26,760 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:27,722 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:28,723 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:29,745 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:30,085 DEBUG SenderThread:1808069 [sender.py:send():379] send: files +2025-01-01 13:54:30,085 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a1afdd9f2c1c0e92713f.mp4 with policy now +2025-01-01 13:54:30,088 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:54:30,088 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:54:30,089 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:54:30,095 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:54:30,330 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:54:30,330 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/2_eval_2_a1afdd9f2c1c0e92713f.mp4 +2025-01-01 13:54:30,330 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval +2025-01-01 13:54:30,367 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/kzsylqqp-media/videos/eval/2_eval_2_a1afdd9f2c1c0e92713f.mp4 +2025-01-01 13:54:30,642 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/m7vbr8rt-media/videos/eval/2_eval_2_a1afdd9f2c1c0e92713f.mp4 +2025-01-01 13:54:30,734 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:31,330 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:54:31,730 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:32,152 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:32,728 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:33,728 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:34,733 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:35,731 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:36,728 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:37,460 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:37,728 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:38,732 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:39,333 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:54:39,684 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:54:39,685 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:54:39,735 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:40,736 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:41,333 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:54:41,728 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:42,731 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:43,334 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:54:43,405 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:43,733 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:43,975 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:54:43,975 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:54:43,976 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:54:44,335 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:54:44,335 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:54:44,591 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/hkgep0ui-events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:54:44,735 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:45,335 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:54:45,740 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:46,739 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:47,736 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:48,696 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:48,736 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:49,735 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:50,738 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:51,337 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:54:51,739 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:52,736 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:53,737 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:53,878 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:54,357 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:54:54,684 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:54:54,684 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:54:54,737 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:55,338 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:54:55,737 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:56,737 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:57,737 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:58,737 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:54:59,696 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:54:59,737 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:00,738 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:01,738 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:02,738 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:03,738 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:04,696 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:04,738 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:05,738 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:06,739 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:07,739 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:08,739 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:09,684 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:55:09,684 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:55:09,740 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:09,788 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:10,742 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:11,740 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:12,740 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:13,741 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:14,741 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:15,696 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:15,741 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:16,741 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:17,742 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:18,743 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:19,741 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:20,741 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:21,696 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:21,742 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:22,742 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:23,742 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:24,357 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:55:24,684 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:55:24,685 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:55:24,742 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:25,349 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:25,742 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:26,742 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:26,786 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:27,746 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:28,696 DEBUG SenderThread:1808069 [sender.py:send():379] send: files +2025-01-01 13:55:28,696 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_48a28150d9236694be84.mp4 with policy now +2025-01-01 13:55:28,699 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:55:28,699 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:55:28,699 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:55:28,703 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:55:28,747 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:29,024 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/oki45pjc-media/videos/eval/3_eval_3_48a28150d9236694be84.mp4 +2025-01-01 13:55:29,350 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:55:29,350 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:29,350 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/3_eval_3_48a28150d9236694be84.mp4 +2025-01-01 13:55:29,350 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval +2025-01-01 13:55:29,644 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/velh0gc1-media/videos/eval/3_eval_3_48a28150d9236694be84.mp4 +2025-01-01 13:55:29,752 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:30,753 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:31,351 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:31,746 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:32,700 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:32,754 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:33,750 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:34,760 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:35,763 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:36,759 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:37,353 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:37,759 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:38,197 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:38,758 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:39,353 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:39,686 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:55:39,687 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:55:39,761 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:40,760 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:41,354 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:41,767 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:42,769 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:43,772 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:44,002 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:44,785 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:45,044 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:55:45,044 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:55:45,045 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:55:45,355 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:55:45,356 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:55:45,601 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/1iis42j0-events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:55:45,770 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:46,781 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:47,356 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:47,774 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:48,774 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:49,045 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:49,793 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:50,793 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:51,357 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:51,776 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:52,776 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:53,358 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:53,777 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:54,358 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:55:54,358 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:54,688 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:55:54,688 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:55:54,777 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:55,359 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:55,776 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:56,776 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:57,360 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:55:57,776 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:58,777 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:55:59,387 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:55:59,776 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:00,776 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:01,776 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:02,777 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:03,777 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:04,705 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:04,777 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:05,363 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:05,777 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:06,777 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:07,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:08,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:09,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:56:09,685 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:56:09,761 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:09,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:10,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:11,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:12,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:13,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:14,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:15,711 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:15,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:16,778 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:17,779 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:18,779 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:19,779 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:20,711 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:20,779 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:21,779 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:22,780 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:23,369 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:23,780 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:24,405 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:56:24,685 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:56:24,685 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:56:24,782 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:25,776 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:25,782 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:26,782 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:27,782 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:28,782 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:29,782 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:30,787 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:30,811 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:31,304 DEBUG SenderThread:1808069 [sender.py:send():379] send: files +2025-01-01 13:56:31,304 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ae99ac981fb1172cf28a.mp4 with policy now +2025-01-01 13:56:31,309 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:56:31,309 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:56:31,309 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:31,309 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:31,372 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:56:31,373 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:31,373 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/4_eval_4_ae99ac981fb1172cf28a.mp4 +2025-01-01 13:56:31,373 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval +2025-01-01 13:56:31,585 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/10vo79tn-media/videos/eval/4_eval_4_ae99ac981fb1172cf28a.mp4 +2025-01-01 13:56:31,783 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:31,861 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/sdxm1gnv-media/videos/eval/4_eval_4_ae99ac981fb1172cf28a.mp4 +2025-01-01 13:56:32,804 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:33,373 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:33,800 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:34,793 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:35,796 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:36,718 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:36,796 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:37,806 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:38,809 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:39,694 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 13:56:39,694 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: stop_status +2025-01-01 13:56:39,816 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:40,820 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:41,376 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:41,813 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:42,009 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:42,816 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:43,819 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:44,826 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:45,063 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 13:56:45,063 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 13:56:45,230 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:56:45,377 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:56:45,377 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:45,377 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 13:56:45,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media +2025-01-01 13:56:45,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table/eval +2025-01-01 13:56:45,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table +2025-01-01 13:56:45,526 INFO wandb-upload_0:1808069 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgll3s0ni +2025-01-01 13:56:45,594 INFO wandb-upload_1:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/gid94xsv-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 13:56:45,629 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/xrvogc08-events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:56:45,700 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 13:56:45,713 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 13:56:45,823 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:45,984 INFO SenderThread:1808069 [sender.py:send_request_log_artifact():1525] logged artifact run-niwmozqn-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDc4MzM2Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI1NTgxMg==', 'latestArtifact': None}} +2025-01-01 13:56:45,985 DEBUG SenderThread:1808069 [sender.py:send():379] send: files +2025-01-01 13:56:45,985 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-01 13:56:45,985 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:56:45,985 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,986 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,986 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:56:45,986 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,990 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,990 DEBUG SenderThread:1808069 [sender.py:send():379] send: telemetry +2025-01-01 13:56:45,990 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,990 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,990 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,991 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,991 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,991 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,991 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,992 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,992 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,992 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,992 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,992 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,993 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,993 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,993 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,993 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,994 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,994 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,994 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,995 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,995 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:45,996 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:45,997 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 13:56:46,240 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/aaxshj2u-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 13:56:46,327 INFO wandb-upload_2:1808069 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppqpe0s0d +2025-01-01 13:56:46,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:56:46,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:56:46,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table/eval/total_success_rate_ci_6_246a86f1a070621a1b8e.table.json +2025-01-01 13:56:46,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table/eval +2025-01-01 13:56:46,707 INFO SenderThread:1808069 [sender.py:send_request_log_artifact():1525] logged artifact run-niwmozqn-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDc4MzM2OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI1NTgxNA==', 'latestArtifact': None}} +2025-01-01 13:56:46,708 DEBUG SenderThread:1808069 [sender.py:send():379] send: files +2025-01-01 13:56:46,708 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_246a86f1a070621a1b8e.table.json with policy now +2025-01-01 13:56:46,710 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:56:46,710 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:46,710 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:46,711 DEBUG SenderThread:1808069 [sender.py:send():379] send: exit +2025-01-01 13:56:46,711 INFO SenderThread:1808069 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 13:56:46,711 INFO SenderThread:1808069 [sender.py:send_exit():588] handling runtime: 321 +2025-01-01 13:56:46,714 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:46,714 INFO SenderThread:1808069 [sender.py:send_exit():594] send defer +2025-01-01 13:56:46,715 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:46,715 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 13:56:46,715 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:46,715 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 13:56:46,715 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 1 +2025-01-01 13:56:46,715 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:46,715 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 13:56:46,715 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:46,715 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 13:56:46,715 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 2 +2025-01-01 13:56:46,715 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:46,715 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 13:56:46,716 INFO HandlerThread:1808069 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 13:56:46,716 DEBUG SystemMonitor:1808069 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 13:56:46,716 DEBUG SystemMonitor:1808069 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 13:56:46,716 INFO HandlerThread:1808069 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 13:56:46,717 INFO HandlerThread:1808069 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 13:56:46,779 INFO HandlerThread:1808069 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 13:56:46,779 INFO HandlerThread:1808069 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 13:56:46,779 INFO HandlerThread:1808069 [interfaces.py:finish():200] Joined network monitor +2025-01-01 13:56:46,780 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:46,780 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 13:56:46,780 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 3 +2025-01-01 13:56:46,780 DEBUG SenderThread:1808069 [sender.py:send():379] send: stats +2025-01-01 13:56:46,780 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:46,780 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 13:56:46,780 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:46,780 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 13:56:46,780 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 4 +2025-01-01 13:56:46,781 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:46,781 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 13:56:46,912 INFO wandb-upload_1:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/ch43452c-media/table/eval/total_success_rate_ci_6_246a86f1a070621a1b8e.table.json +2025-01-01 13:56:47,171 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/eev4frtl-media/table/eval/total_success_rate_ci_6_246a86f1a070621a1b8e.table.json +2025-01-01 13:56:47,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:56:47,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:48,378 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/config.yaml +2025-01-01 13:56:49,379 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:53,121 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:56:53,121 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:53,121 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 13:56:53,121 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:53,122 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:53,122 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 13:56:53,122 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 5 +2025-01-01 13:56:53,122 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:56:53,123 DEBUG SenderThread:1808069 [sender.py:send():379] send: history +2025-01-01 13:56:53,123 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: summary_record +2025-01-01 13:56:53,123 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:53,123 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 13:56:53,124 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:53,124 DEBUG SenderThread:1808069 [sender.py:send():379] send: summary +2025-01-01 13:56:53,124 INFO SenderThread:1808069 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 13:56:53,124 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:53,124 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 13:56:53,124 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 6 +2025-01-01 13:56:53,125 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:53,125 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 13:56:53,125 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:53,125 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 13:56:53,125 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 7 +2025-01-01 13:56:53,125 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 13:56:53,125 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:53,125 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 13:56:53,125 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:53,125 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 13:56:53,230 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:56:53,381 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:56:55,369 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 8 +2025-01-01 13:56:55,369 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:56:55,369 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:55,369 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 13:56:55,369 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:55,370 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 13:56:55,370 INFO SenderThread:1808069 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 13:56:55,374 INFO SenderThread:1808069 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 13:56:55,376 INFO SenderThread:1808069 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 13:56:55,381 INFO Thread-12 :1808069 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:55,386 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 9 +2025-01-01 13:56:55,386 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:55,386 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 13:56:55,386 DEBUG SenderThread:1808069 [sender.py:send():379] send: artifact +2025-01-01 13:56:55,551 INFO SenderThread:1808069 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIxODYyMQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 13:56:55,551 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:55,551 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 13:56:55,551 INFO SenderThread:1808069 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 13:56:56,232 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:56:56,382 INFO SenderThread:1808069 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files +2025-01-01 13:56:56,382 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/config.yaml config.yaml +2025-01-01 13:56:56,383 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json wandb-summary.json +2025-01-01 13:56:56,383 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:56:56,383 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/requirements.txt requirements.txt +2025-01-01 13:56:56,383 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-metadata.json wandb-metadata.json +2025-01-01 13:56:56,383 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log output.log +2025-01-01 13:56:56,383 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/diff.patch diff.patch +2025-01-01 13:56:56,386 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 13:56:56,386 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/3_eval_3_48a28150d9236694be84.mp4 media/videos/eval/3_eval_3_48a28150d9236694be84.mp4 +2025-01-01 13:56:56,386 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/4_eval_4_ae99ac981fb1172cf28a.mp4 media/videos/eval/4_eval_4_ae99ac981fb1172cf28a.mp4 +2025-01-01 13:56:56,388 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/2_eval_2_a1afdd9f2c1c0e92713f.mp4 media/videos/eval/2_eval_2_a1afdd9f2c1c0e92713f.mp4 +2025-01-01 13:56:56,388 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/1_eval_1_9e0e7530b69fb1fd581f.mp4 media/videos/eval/1_eval_1_9e0e7530b69fb1fd581f.mp4 +2025-01-01 13:56:56,388 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/videos/eval/0_eval_0_294304c5db5cd37da633.mp4 media/videos/eval/0_eval_0_294304c5db5cd37da633.mp4 +2025-01-01 13:56:56,391 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table/eval/total_success_rate_ci_6_246a86f1a070621a1b8e.table.json media/table/eval/total_success_rate_ci_6_246a86f1a070621a1b8e.table.json +2025-01-01 13:56:56,391 INFO SenderThread:1808069 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-01 13:56:56,391 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 10 +2025-01-01 13:56:56,391 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:56:56,391 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:56,391 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 13:56:56,392 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:56,392 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 13:56:56,392 INFO SenderThread:1808069 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 13:56:56,608 INFO wandb-upload_3:1808069 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/requirements.txt +2025-01-01 13:56:56,609 INFO wandb-upload_4:1808069 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/output.log +2025-01-01 13:56:56,610 INFO wandb-upload_0:1808069 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/wandb-summary.json +2025-01-01 13:56:56,610 INFO wandb-upload_1:1808069 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/files/config.yaml +2025-01-01 13:56:56,629 INFO wandb-upload_2:1808069 [upload_job.py:push():130] Uploaded file /tmp/tmplgo436apwandb/6ye2f0b5-events.out.tfevents.1735757484.leonmkim-ROG-Strix-G15CS-G15CS.1807895.0 +2025-01-01 13:56:56,829 INFO Thread-11 (_thread_body):1808069 [sender.py:transition_state():614] send defer: 11 +2025-01-01 13:56:56,829 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:56,829 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 13:56:56,830 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:56,830 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 13:56:56,830 INFO SenderThread:1808069 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 13:56:56,831 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 12 +2025-01-01 13:56:56,831 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:56,831 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 13:56:56,831 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:56,831 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 13:56:56,831 INFO SenderThread:1808069 [file_stream.py:finish():601] file stream finish called +2025-01-01 13:56:57,070 INFO SenderThread:1808069 [file_stream.py:finish():605] file stream finish is done +2025-01-01 13:56:57,070 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 13 +2025-01-01 13:56:57,070 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:57,070 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 13:56:57,070 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:57,070 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 13:56:57,070 INFO SenderThread:1808069 [sender.py:transition_state():614] send defer: 14 +2025-01-01 13:56:57,070 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: defer +2025-01-01 13:56:57,070 INFO HandlerThread:1808069 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 13:56:57,071 DEBUG SenderThread:1808069 [sender.py:send():379] send: final +2025-01-01 13:56:57,071 DEBUG SenderThread:1808069 [sender.py:send():379] send: footer +2025-01-01 13:56:57,071 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: defer +2025-01-01 13:56:57,071 INFO SenderThread:1808069 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 13:56:57,071 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:56:57,071 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:56:57,072 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 13:56:57,072 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 13:56:57,072 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: server_info +2025-01-01 13:56:57,074 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 13:56:57,074 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 13:56:57,075 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 13:56:57,136 DEBUG SenderThread:1808069 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 13:56:57,136 INFO MainThread:1808069 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 13:56:57,137 INFO MainThread:1808069 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 13:56:57,137 INFO MainThread:1808069 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 13:56:57,137 DEBUG HandlerThread:1808069 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 13:56:57,137 INFO HandlerThread:1808069 [handler.py:finish():882] shutting down handler +2025-01-01 13:56:58,074 INFO WriterThread:1808069 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/run-niwmozqn.wandb +2025-01-01 13:56:58,136 INFO SenderThread:1808069 [sender.py:finish():1615] shutting down sender +2025-01-01 13:56:58,136 INFO SenderThread:1808069 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 13:56:58,136 INFO SenderThread:1808069 [file_pusher.py:join():175] waiting for file pusher diff --git a/niwmozqn/wandb/run-20250101_135124-niwmozqn/logs/debug.log b/niwmozqn/wandb/run-20250101_135124-niwmozqn/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..224d1ba36a0c950c55406476bb5426aa32a5eef8 --- /dev/null +++ b/niwmozqn/wandb/run-20250101_135124-niwmozqn/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_setup.py:_flush():76] Configure stats pid to 1807895 +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/settings +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/logs/debug.log +2025-01-01 13:51:24,026 INFO MainThread:1807895 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/wandb/run-20250101_135124-niwmozqn/logs/debug-internal.log +2025-01-01 13:51:24,027 INFO MainThread:1807895 [wandb_init.py:init():569] calling init triggers +2025-01-01 13:51:24,027 INFO MainThread:1807895 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1745_0', 'wandb_notes': '1745_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn'} +2025-01-01 13:51:24,027 INFO MainThread:1807895 [wandb_init.py:init():619] starting backend +2025-01-01 13:51:24,027 INFO MainThread:1807895 [wandb_init.py:init():623] setting up manager +2025-01-01 13:51:24,032 INFO MainThread:1807895 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-01 13:51:24,033 INFO MainThread:1807895 [wandb_init.py:init():631] backend started and connected +2025-01-01 13:51:24,046 INFO MainThread:1807895 [wandb_init.py:init():720] updated telemetry +2025-01-01 13:51:24,055 INFO MainThread:1807895 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-01 13:51:24,271 INFO MainThread:1807895 [wandb_run.py:_on_init():2435] communicating current version +2025-01-01 13:51:24,341 INFO MainThread:1807895 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-01 13:51:24,341 INFO MainThread:1807895 [wandb_init.py:init():804] starting run threads in backend +2025-01-01 13:51:24,682 INFO MainThread:1807895 [wandb_run.py:_console_start():2413] atexit reg +2025-01-01 13:51:24,682 INFO MainThread:1807895 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-01 13:51:24,682 INFO MainThread:1807895 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-01 13:51:24,682 INFO MainThread:1807895 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-01 13:51:24,686 INFO MainThread:1807895 [wandb_init.py:init():847] run started, returning control to user process +2025-01-01 13:51:24,686 INFO MainThread:1807895 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/niwmozqn/tb, True +2025-01-01 13:51:30,015 INFO MainThread:1807895 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-01 13:56:58,138 WARNING MsgRouterThr:1807895 [router.py:message_loop():77] message_loop has been closed diff --git a/niwmozqn/wandb/run-20250101_135124-niwmozqn/run-niwmozqn.wandb b/niwmozqn/wandb/run-20250101_135124-niwmozqn/run-niwmozqn.wandb new file mode 100644 index 0000000000000000000000000000000000000000..3b931433ac82fb1f0d83713541d96bc924b0caaa Binary files /dev/null and b/niwmozqn/wandb/run-20250101_135124-niwmozqn/run-niwmozqn.wandb differ diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..75b711b395eeca61e62a4c8d6c6f1c7fd4780edd --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18061beab4a49967447d947a2f10df2df6f8729a8c62860689780d806ad67a69 +size 910217934 diff --git a/snapshot_12749.pt b/snapshot_12749.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef3add27756407704f2c8a96b68cdbd718e34ca2 --- /dev/null +++ b/snapshot_12749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08c849e469e510eada60e3dd84955e74f6948304415ec5ffb5b694aa44132794 +size 910217934 diff --git a/snapshot_13499.pt b/snapshot_13499.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f007f220c42c5537701be6235d0edc4b2337b11 --- /dev/null +++ b/snapshot_13499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:663ad987638ed8985a4c6835aa8421852bf1c77d0298b6c30e726def5848c8e7 +size 910217934 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..28ef66dcc8c332047926f944112ddbcbbf646035 --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86315c890fe96da46365f59d236479d9e39e9b5bf3218c7d98958169738ed024 +size 910217934 diff --git a/snapshot_14999.pt b/snapshot_14999.pt new file mode 100644 index 0000000000000000000000000000000000000000..dadb4ddb62d4b0910e0d362b3154679b2f2d08a4 --- /dev/null +++ b/snapshot_14999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a10f4ae4780cb06dcd87b8c056a437a39f3aea278326b27455035c157e285a8 +size 910217934 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..333706f1c1c25a3a8ab4d9089ae8fa29d1e2ef17 --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5673083616f12ebbe048ebfa0dcdbffcbd4e1175ea62e66e42e38d79fb0fccaa +size 910217934 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..10034eb6366edb4f0f08c7c9f9daedac0dff94a8 --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ac4de8eb6ecb8b3fc1d316b6c01d303e8d7d198750f45faad888d8efca5b7e7 +size 910217488 diff --git a/snapshot_7499.pt b/snapshot_7499.pt new file mode 100644 index 0000000000000000000000000000000000000000..833119a5155635b7ae0fa278b4e519b7c3ad3782 --- /dev/null +++ b/snapshot_7499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e86653dfa75eba30363af96c7e3dafb99b7f34969c36e9cdcf04d520e8a7e3e4 +size 910217934 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..89191beeb08fa946cb8a12be3982a26e0911d0e1 --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e10acc2f72840d1ba228dd5c0c3ff0a6c4e08d7385c2a28e907389a1b7d1eb8d +size 910217934 diff --git a/snapshot_8749.pt b/snapshot_8749.pt new file mode 100644 index 0000000000000000000000000000000000000000..63adc5d10af440a54ef9dd50c8b868fa442a61aa --- /dev/null +++ b/snapshot_8749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e11b455c73113dcd07fdc66b56a88b6ebe2e96e38070442dac52eb1756658368 +size 910217934 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..16f886667cc37708294b48d5e6cc66bdbbef6f3a --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04d95cb711cde3deccc6dfd460fa943544aaeae7fb0f72019ccdd9da7f05da9e +size 910217934 diff --git a/snapshot_9249.pt b/snapshot_9249.pt new file mode 100644 index 0000000000000000000000000000000000000000..8da2be55325d14dea218b1b30a7eaa714310b68e --- /dev/null +++ b/snapshot_9249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44192f24ec2bd9052e097b2661dfa1c33e50c1ef5fcfd75b086016b2c5ae919e +size 910217934 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..333706f1c1c25a3a8ab4d9089ae8fa29d1e2ef17 --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5673083616f12ebbe048ebfa0dcdbffcbd4e1175ea62e66e42e38d79fb0fccaa +size 910217934